From 5690fd0d3304f378754b23b098bd7cb5f4aa1976 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 14:38:21 +0200 Subject: [PATCH 0001/1392] initial commit with latest version extracted from git-python --- .gitignore | 1 + __init__.py | 6 + db.py | 341 +++++++++++++++++++++++++++++++++ fun.py | 115 ++++++++++++ stream.py | 445 ++++++++++++++++++++++++++++++++++++++++++++ test/__init__.py | 1 + test/lib.py | 60 ++++++ test/test_db.py | 90 +++++++++ test/test_stream.py | 172 +++++++++++++++++ test/test_utils.py | 15 ++ utils.py | 38 ++++ 11 files changed, 1284 insertions(+) create mode 100644 .gitignore create mode 100644 __init__.py create mode 100644 db.py create mode 100644 fun.py create mode 100644 stream.py create mode 100644 test/__init__.py create mode 100644 test/lib.py create mode 100644 test/test_db.py create mode 100644 test/test_stream.py create mode 100644 test/test_utils.py create mode 100644 utils.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..0d20b6487 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/__init__.py b/__init__.py new file mode 100644 index 000000000..5789d7eb7 --- /dev/null +++ b/__init__.py @@ -0,0 +1,6 @@ +"""Initialize the object database module""" + +# default imports +from db import * +from stream import * + diff --git a/db.py b/db.py new file mode 100644 index 000000000..5d3cc6a3f --- /dev/null +++ b/db.py @@ -0,0 +1,341 @@ +"""Contains implementations of database retrieveing objects""" +from git.utils import IndexFileSHA1Writer +from git.errors import ( + InvalidDBRoot, + BadObject, + BadObjectType + ) + +from stream import ( + DecompressMemMapReader, + FDCompressedSha1Writer, + Sha1Writer, + OStream, + OInfo + ) + +from utils import ( + ENOENT, + to_hex_sha, + exists, + hex_to_bin, + isdir, + mkdir, + rename, + dirname, + join + ) + +from fun import ( + chunk_size, + loose_object_header_info, + write_object, + stream_copy + ) + +import tempfile +import mmap +import os + + +__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'LooseObjectDB', 'PackedDB', + 'CompoundDB', 'ReferenceDB', 'GitObjectDB' ) + +class ObjectDBR(object): + """Defines an interface for object database lookup. + Objects are identified either by hex-sha (40 bytes) or + by sha (20 bytes)""" + + def __contains__(self, sha): + return self.has_obj + + #{ Query Interface + def has_object(self, sha): + """ + :return: True if the object identified by the given 40 byte hexsha or 20 bytes + binary sha is contained in the database + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def info(self, sha): + """ :return: OInfo instance + :param sha: 40 bytes hexsha or 20 bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def info_async(self, input_channel): + """Retrieve information of a multitude of objects asynchronously + :param input_channel: Channel yielding the sha's of the objects of interest + :return: Channel yielding OInfo|InvalidOInfo, in any order""" + raise NotImplementedError("To be implemented in subclass") + + def stream(self, sha): + """:return: OStream instance + :param sha: 40 bytes hexsha or 20 bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def stream_async(self, input_channel): + """Retrieve the OStream of multiple objects + :param input_channel: see ``info`` + :param max_threads: see ``ObjectDBW.store`` + :return: Channel yielding OStream|InvalidOStream instances in any order""" + raise NotImplementedError("To be implemented in subclass") + + #} END query interface + +class ObjectDBW(object): + """Defines an interface to create objects in the database""" + + def __init__(self, *args, **kwargs): + self._ostream = None + + #{ Edit Interface + def set_ostream(self, stream): + """Adjusts the stream to which all data should be sent when storing new objects + :param stream: if not None, the stream to use, if None the default stream + will be used. + :return: previously installed stream, or None if there was no override + :raise TypeError: if the stream doesn't have the supported functionality""" + cstream = self._ostream + self._ostream = stream + return cstream + + def ostream(self): + """:return: overridden output stream this instance will write to, or None + if it will write to the default stream""" + return self._ostream + + def store(self, istream): + """Create a new object in the database + :return: the input istream object with its sha set to its corresponding value + :param istream: IStream compatible instance. If its sha is already set + to a value, the object will just be stored in the our database format, + in which case the input stream is expected to be in object format ( header + contents ). + :raise IOError: if data could not be written""" + raise NotImplementedError("To be implemented in subclass") + + def store_async(self, input_channel): + """Create multiple new objects in the database asynchronously. The method will + return right away, returning an output channel which receives the results as + they are computed. + + :return: Channel yielding your IStream which served as input, in any order. + The IStreams sha will be set to the sha it received during the process, + or its error attribute will be set to the exception informing about the error. + :param input_channel: Channel yielding IStream instance. + As the same instances will be used in the output channel, you can create a map + between the id(istream) -> istream + :note:As some ODB implementations implement this operation as atomic, they might + abort the whole operation if one item could not be processed. Hence check how + many items have actually been produced.""" + raise NotImplementedError("To be implemented in subclass") + + #} END edit interface + + +class FileDBBase(object): + """Provides basic facilities to retrieve files of interest, including + caching facilities to help mapping hexsha's to objects""" + + def __init__(self, root_path): + """Initialize this instance to look for its files at the given root path + All subsequent operations will be relative to this path + :raise InvalidDBRoot: + :note: The base will not perform any accessablity checking as the base + might not yet be accessible, but become accessible before the first + access.""" + super(FileDBBase, self).__init__() + self._root_path = root_path + + + #{ Interface + def root_path(self): + """:return: path at which this db operates""" + return self._root_path + + def db_path(self, rela_path): + """ + :return: the given relative path relative to our database root, allowing + to pontentially access datafiles""" + return join(self._root_path, rela_path) + #} END interface + + + +class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): + """A database which operates on loose object files""" + + # CONFIGURATION + # chunks in which data will be copied between streams + stream_chunk_size = chunk_size + + + def __init__(self, root_path): + super(LooseObjectDB, self).__init__(root_path) + self._hexsha_to_file = dict() + # Additional Flags - might be set to 0 after the first failure + # Depending on the root, this might work for some mounts, for others not, which + # is why it is per instance + self._fd_open_flags = getattr(os, 'O_NOATIME', 0) + + #{ Interface + def object_path(self, hexsha): + """ + :return: path at which the object with the given hexsha would be stored, + relative to the database root""" + return join(hexsha[:2], hexsha[2:]) + + def readable_db_object_path(self, hexsha): + """ + :return: readable object path to the object identified by hexsha + :raise BadObject: If the object file does not exist""" + try: + return self._hexsha_to_file[hexsha] + except KeyError: + pass + # END ignore cache misses + + # try filesystem + path = self.db_path(self.object_path(hexsha)) + if exists(path): + self._hexsha_to_file[hexsha] = path + return path + # END handle cache + raise BadObject(hexsha) + + #} END interface + + def _map_loose_object(self, sha): + """ + :return: memory map of that file to allow random read access + :raise BadObject: if object could not be located""" + db_path = self.db_path(self.object_path(to_hex_sha(sha))) + try: + fd = os.open(db_path, os.O_RDONLY|self._fd_open_flags) + except OSError,e: + if e.errno != ENOENT: + # try again without noatime + try: + fd = os.open(db_path, os.O_RDONLY) + except OSError: + raise BadObject(to_hex_sha(sha)) + # didn't work because of our flag, don't try it again + self._fd_open_flags = 0 + else: + raise BadObject(to_hex_sha(sha)) + # END handle error + # END exception handling + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + finally: + os.close(fd) + # END assure file is closed + + def set_ostream(self, stream): + """:raise TypeError: if the stream does not support the Sha1Writer interface""" + if stream is not None and not isinstance(stream, Sha1Writer): + raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) + return super(LooseObjectDB, self).set_ostream(stream) + + def info(self, sha): + m = self._map_loose_object(sha) + try: + type, size = loose_object_header_info(m) + return OInfo(sha, type, size) + finally: + m.close() + # END assure release of system resources + + def stream(self, sha): + m = self._map_loose_object(sha) + type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) + return OStream(sha, type, size, stream) + + def has_object(self, sha): + try: + self.readable_db_object_path(to_hex_sha(sha)) + return True + except BadObject: + return False + # END check existance + + def store(self, istream): + """note: The sha we produce will be hex by nature""" + tmp_path = None + writer = self.ostream() + if writer is None: + # open a tmp file to write the data to + fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) + writer = FDCompressedSha1Writer(fd) + # END handle custom writer + + try: + try: + if istream.sha is not None: + stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size) + else: + # write object with header, we have to make a new one + write_object(istream.type, istream.size, istream.read, writer.write, + chunk_size=self.stream_chunk_size) + # END handle direct stream copies + except: + if tmp_path: + os.remove(tmp_path) + raise + # END assure tmpfile removal on error + finally: + if tmp_path: + writer.close() + # END assure target stream is closed + + sha = istream.sha or writer.sha(as_hex=True) + + if tmp_path: + obj_path = self.db_path(self.object_path(sha)) + obj_dir = dirname(obj_path) + if not isdir(obj_dir): + mkdir(obj_dir) + # END handle destination directory + rename(tmp_path, obj_path) + # END handle dry_run + + istream.sha = sha + return istream + + +class PackedDB(FileDBBase, ObjectDBR): + """A database operating on a set of object packs""" + + +class CompoundDB(ObjectDBR): + """A database which delegates calls to sub-databases""" + + +class ReferenceDB(CompoundDB): + """A database consisting of database referred to in a file""" + + +#class GitObjectDB(CompoundDB, ObjectDBW): +class GitObjectDB(LooseObjectDB): + """A database representing the default git object store, which includes loose + objects, pack files and an alternates file + + It will create objects only in the loose object database. + :note: for now, we use the git command to do all the lookup, just until he + have packs and the other implementations + """ + def __init__(self, root_path, git): + """Initialize this instance with the root and a git command""" + super(GitObjectDB, self).__init__(root_path) + self._git = git + + def info(self, sha): + t = self._git.get_object_header(sha) + return OInfo(*t) + + def stream(self, sha): + """For now, all lookup is done by git itself""" + t = self._git.stream_object_data(sha) + return OStream(*t) + diff --git a/fun.py b/fun.py new file mode 100644 index 000000000..3321a8ea4 --- /dev/null +++ b/fun.py @@ -0,0 +1,115 @@ +"""Contains basic c-functions which usually contain performance critical code +Keeping this code separate from the beginning makes it easier to out-source +it into c later, if required""" + +from git.errors import ( + BadObjectType + ) + +import zlib +decompressobj = zlib.decompressobj + + +# INVARIANTS +type_id_to_type_map = { + 1 : "commit", + 2 : "tree", + 3 : "blob", + 4 : "tag" + } + +# used when dealing with larger streams +chunk_size = 1000*1000 + +__all__ = ('is_loose_object', 'loose_object_header_info', 'object_header_info', + 'write_object' ) + +#{ Routines + +def is_loose_object(m): + """:return: True the file contained in memory map m appears to be a loose object. + Only the first two bytes are needed""" + b0, b1 = map(ord, m[:2]) + word = (b0 << 8) + b1 + return b0 == 0x78 and (word % 31) == 0 + +def loose_object_header_info(m): + """:return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + object as well as its uncompressed size in bytes. + :param m: memory map from which to read the compressed object data""" + decompress_size = 8192 # is used in cgit as well + hdr = decompressobj().decompress(m, decompress_size) + type_name, size = hdr[:hdr.find("\0")].split(" ") + return type_name, int(size) + +def object_header_info(m): + """:return: tuple(type_string, uncompressed_size_in_bytes + :param mmap: mapped memory map. It will be + seeked to the actual start of the object contents, which can be used + to initialize a zlib decompress object. + :note: This routine can only handle new-style objects which are assumably contained + in packs + """ + assert not is_loose_object(m), "Use loose_object_header_info instead" + + c = b0 # first byte + i = 1 # next char to read + type_id = (c >> 4) & 7 # numeric type + size = c & 15 # starting size + s = 4 # starting bit-shift size + while c & 0x80: + c = ord(m[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + + # finally seek the map to the start of the data stream + m.seek(i) + try: + return (type_id_to_type_map[type_id], size) + except KeyError: + # invalid object type - we could try to be smart now and decode part + # of the stream to get the info, problem is that we had trouble finding + # the exact start of the content stream + raise BadObjectType(type_id) + # END handle exceptions + +def write_object(type, size, read, write, chunk_size=chunk_size): + """Write the object as identified by type, size and source_stream into the + target_stream + + :param type: type string of the object + :param size: amount of bytes to write from source_stream + :param read: read method of a stream providing the content data + :param write: write method of the output stream + :param close_target_stream: if True, the target stream will be closed when + the routine exits, even if an error is thrown + :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" + tbw = 0 # total num bytes written + + # WRITE HEADER: type SP size NULL + tbw += write("%s %i\0" % (type, size)) + tbw += stream_copy(read, write, size, chunk_size) + + return tbw + +def stream_copy(read, write, size, chunk_size): + """Copy a stream up to size bytes using the provided read and write methods, + in chunks of chunk_size + :note: its much like stream_copy utility, but operates just using methods""" + dbw = 0 # num data bytes written + + # WRITE ALL DATA UP TO SIZE + while True: + cs = min(chunk_size, size-dbw) + data_len = write(read(cs)) + dbw += data_len + if data_len < cs or dbw == size: + break + # END check for stream end + # END duplicate data + return dbw + + +#} END routines diff --git a/stream.py b/stream.py new file mode 100644 index 000000000..da97cf5b6 --- /dev/null +++ b/stream.py @@ -0,0 +1,445 @@ +import zlib +from cStringIO import StringIO +from git.utils import make_sha +import errno + +from utils import ( + to_hex_sha, + to_bin_sha, + write, + close + ) + +__all__ = ('OInfo', 'OStream', 'IStream', 'InvalidOInfo', 'InvalidOStream', + 'DecompressMemMapReader', 'FDCompressedSha1Writer') + + +# ZLIB configuration +# used when compressing objects - 1 to 9 ( slowest ) +Z_BEST_SPEED = 1 + + +#{ ODB Bases + +class OInfo(tuple): + """Carries information about an object in an ODB, provdiing information + about the sha of the object, the type_string as well as the uncompressed size + in bytes. + + It can be accessed using tuple notation and using attribute access notation:: + + assert dbi[0] == dbi.sha + assert dbi[1] == dbi.type + assert dbi[2] == dbi.size + + The type is designed to be as lighteight as possible.""" + __slots__ = tuple() + + def __new__(cls, sha, type, size): + return tuple.__new__(cls, (sha, type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + @property + def sha(self): + return self[0] + + @property + def type(self): + return self[1] + + @property + def size(self): + return self[2] + #} END interface + + +class OStream(OInfo): + """Base for object streams retrieved from the database, providing additional + information about the stream. + Generally, ODB streams are read-only as objects are immutable""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + + def __init__(self, *args, **kwargs): + tuple.__init__(self) + + #{ Stream Reader Interface + + def read(self, size=-1): + return self[3].read(size) + + #} END stream reader interface + + +class IStream(list): + """Represents an input content stream to be fed into the ODB. It is mutable to allow + the ODB to record information about the operations outcome right in this instance. + + It provides interfaces for the OStream and a StreamReader to allow the instance + to blend in without prior conversion. + + The only method your content stream must support is 'read'""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, sha=None): + return list.__new__(cls, (sha, type, size, stream, None)) + + def __init__(self, type, size, stream, sha=None): + list.__init__(self, (sha, type, size, stream, None)) + + #{ Interface + + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return to_hex_sha(self[0]) + + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return to_bin_sha(self[0]) + + def _error(self): + """:return: the error that occurred when processing the stream, or None""" + return self[4] + + def _set_error(self, exc): + """Set this input stream to the given exc, may be None to reset the error""" + self[4] = exc + + error = property(_error, _set_error) + + #} END interface + + #{ Stream Reader Interface + + def read(self, size=-1): + """Implements a simple stream reader interface, passing the read call on + to our internal stream""" + return self[3].read(size) + + #} END stream reader interface + + #{ interface + + def _set_sha(self, sha): + self[0] = sha + + def _sha(self): + return self[0] + + sha = property(_sha, _set_sha) + + + def _type(self): + return self[1] + + def _set_type(self, type): + self[1] = type + + type = property(_type, _set_type) + + def _size(self): + return self[2] + + def _set_size(self, size): + self[2] = size + + size = property(_size, _set_size) + + def _stream(self): + return self[3] + + def _set_stream(self, stream): + self[3] = stream + + stream = property(_stream, _set_stream) + + #} END odb info interface + + +class InvalidOInfo(tuple): + """Carries information about a sha identifying an object which is invalid in + the queried database. The exception attribute provides more information about + the cause of the issue""" + __slots__ = tuple() + + def __new__(cls, sha, exc): + return tuple.__new__(cls, (sha, exc)) + + def __init__(self, sha, exc): + tuple.__init__(self, (sha, exc)) + + @property + def sha(self): + return self[0] + + @property + def error(self): + """:return: exception instance explaining the failure""" + return self[1] + + +class InvalidOStream(InvalidOInfo): + """Carries information about an invalid ODB stream""" + __slots__ = tuple() + +#} END ODB Bases + + +#{ RO Streams + +class DecompressMemMapReader(object): + """Reads data in chunks from a memory map and decompresses it. The client sees + only the uncompressed data, respective file-like read calls are handling on-demand + buffered decompression accordingly + + A constraint on the total size of bytes is activated, simulating + a logical file within a possibly larger physical memory area + + To read efficiently, you clearly don't want to read individual bytes, instead, + read a few kilobytes at least. + + :note: The chunk-size should be carefully selected as it will involve quite a bit + of string copying due to the way the zlib is implemented. Its very wasteful, + hence we try to find a good tradeoff between allocation time and number of + times we actually allocate. An own zlib implementation would be good here + to better support streamed reading - it would only need to keep the mmap + and decompress it into chunks, thats all ... """ + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close') + + max_read_size = 512*1024 # currently unused + + def __init__(self, m, close_on_deletion, size): + """Initialize with mmap for stream reading + :param m: must be content data - use new if you have object data and no size""" + self._m = m + self._zip = zlib.decompressobj() + self._buf = None # buffer of decompressed bytes + self._buflen = 0 # length of bytes in buffer + self._s = size # size of uncompressed data to read in total + self._br = 0 # num uncompressed bytes read + self._cws = 0 # start byte of compression window + self._cwe = 0 # end byte of compression window + self._close = close_on_deletion # close the memmap on deletion ? + + def __del__(self): + if self._close: + self._m.close() + # END handle resource freeing + + def _parse_header_info(self): + """If this stream contains object data, parse the header info and skip the + stream to a point where each read will yield object content + :return: parsed type_string, size""" + # read header + maxb = 512 # should really be enough, cgit uses 8192 I believe + self._s = maxb + hdr = self.read(maxb) + hdrend = hdr.find("\0") + type, size = hdr[:hdrend].split(" ") + size = int(size) + self._s = size + + # adjust internal state to match actual header length that we ignore + # The buffer will be depleted first on future reads + self._br = 0 + hdrend += 1 # count terminating \0 + self._buf = StringIO(hdr[hdrend:]) + self._buflen = len(hdr) - hdrend + + return type, size + + @classmethod + def new(self, m, close_on_deletion=False): + """Create a new DecompressMemMapReader instance for acting as a read-only stream + This method parses the object header from m and returns the parsed + type and size, as well as the created stream instance. + :param m: memory map on which to oparate. It must be object data ( header + contents ) + :param close_on_deletion: if True, the memory map will be closed once we are + being deleted""" + inst = DecompressMemMapReader(m, close_on_deletion, 0) + type, size = inst._parse_header_info() + return type, size, inst + + def read(self, size=-1): + if size < 1: + size = self._s - self._br + else: + size = min(size, self._s - self._br) + # END clamp size + + if size == 0: + return str() + # END handle depletion + + # protect from memory peaks + # If he tries to read large chunks, our memory patterns get really bad + # as we end up copying a possibly huge chunk from our memory map right into + # memory. This might not even be possible. Nonetheless, try to dampen the + # effect a bit by reading in chunks, returning a huge string in the end. + # Our performance now depends on StringIO. This way we don't need two large + # buffers in peak times, but only one large one in the end which is + # the return buffer + # NO: We don't do it - if the user thinks its best, he is right. If he + # has trouble, he will start reading in chunks. According to our tests + # its still faster if we read 10 Mb at once instead of chunking it. + + # if size > self.max_read_size: + # sio = StringIO() + # while size: + # read_size = min(self.max_read_size, size) + # data = self.read(read_size) + # sio.write(data) + # size -= len(data) + # if len(data) < read_size: + # break + # # END data loop + # sio.seek(0) + # return sio.getvalue() + # # END handle maxread + # + # deplete the buffer, then just continue using the decompress object + # which has an own buffer. We just need this to transparently parse the + # header from the zlib stream + dat = str() + if self._buf: + if self._buflen >= size: + # have enough data + dat = self._buf.read(size) + self._buflen -= size + self._br += size + return dat + else: + dat = self._buf.read() # ouch, duplicates data + size -= self._buflen + self._br += self._buflen + + self._buflen = 0 + self._buf = None + # END handle buffer len + # END handle buffer + + # decompress some data + # Abstract: zlib needs to operate on chunks of our memory map ( which may + # be large ), as it will otherwise and always fill in the 'unconsumed_tail' + # attribute which possible reads our whole map to the end, forcing + # everything to be read from disk even though just a portion was requested. + # As this would be a nogo, we workaround it by passing only chunks of data, + # moving the window into the memory map along as we decompress, which keeps + # the tail smaller than our chunk-size. This causes 'only' the chunk to be + # copied once, and another copy of a part of it when it creates the unconsumed + # tail. We have to use it to hand in the appropriate amount of bytes durin g + # the next read. + tail = self._zip.unconsumed_tail + if tail: + # move the window, make it as large as size demands. For code-clarity, + # we just take the chunk from our map again instead of reusing the unconsumed + # tail. The latter one would safe some memory copying, but we could end up + # with not getting enough data uncompressed, so we had to sort that out as well. + # Now we just assume the worst case, hence the data is uncompressed and the window + # needs to be as large as the uncompressed bytes we want to read. + self._cws = self._cwe - len(tail) + self._cwe = self._cws + size + else: + cws = self._cws + self._cws = self._cwe + self._cwe = cws + size + # END handle tail + + + # if window is too small, make it larger so zip can decompress something + win_size = self._cwe - self._cws + if win_size < 8: + self._cwe = self._cws + 8 + # END adjust winsize + indata = self._m[self._cws:self._cwe] # another copy ... :( + + # get the actual window end to be sure we don't use it for computations + self._cwe = self._cws + len(indata) + + dcompdat = self._zip.decompress(indata, size) + + self._br += len(dcompdat) + if dat: + dcompdat = dat + dcompdat + + return dcompdat + +#} END RO streams + + +#{ W Streams + +class Sha1Writer(object): + """Simple stream writer which produces a sha whenever you like as it degests + everything it is supposed to write""" + __slots__ = "sha1" + + def __init__(self): + self.sha1 = make_sha("") + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: lenght of incoming data""" + self.sha1.update(data) + return len(data) + + # END stream interface + + #{ Interface + + def sha(self, as_hex = False): + """:return: sha so far + :param as_hex: if True, sha will be hex-encoded, binary otherwise""" + if as_hex: + return self.sha1.hexdigest() + return self.sha1.digest() + + #} END interface + +class FDCompressedSha1Writer(Sha1Writer): + """Digests data written to it, making the sha available, then compress the + data and write it to the file descriptor + :note: operates on raw file descriptors + :note: for this to work, you have to use the close-method of this instance""" + __slots__ = ("fd", "sha1", "zip") + + # default exception + exc = IOError("Failed to write all bytes to filedescriptor") + + def __init__(self, fd): + super(FDCompressedSha1Writer, self).__init__() + self.fd = fd + self.zip = zlib.compressobj(Z_BEST_SPEED) + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: lenght of incoming data""" + self.sha1.update(data) + cdata = self.zip.compress(data) + bytes_written = write(self.fd, cdata) + if bytes_written != len(cdata): + raise self.exc + return len(data) + + def close(self): + remainder = self.zip.flush() + if write(self.fd, remainder) != len(remainder): + raise self.exc + return close(self.fd) + + #} END stream interface + +#} END W streams diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/test/__init__.py @@ -0,0 +1 @@ + diff --git a/test/lib.py b/test/lib.py new file mode 100644 index 000000000..d51997488 --- /dev/null +++ b/test/lib.py @@ -0,0 +1,60 @@ +"""Utilities used in ODB testing""" +from git.odb import ( + OStream, + ) +from git.odb.stream import Sha1Writer + +import zlib +from cStringIO import StringIO + +#{ Stream Utilities + +class DummyStream(object): + def __init__(self): + self.was_read = False + self.bytes = 0 + self.closed = False + + def read(self, size): + self.was_read = True + self.bytes = size + + def close(self): + self.closed = True + + def _assert(self): + assert self.was_read + + +class DeriveTest(OStream): + def __init__(self, sha, type, size, stream, *args, **kwargs): + self.myarg = kwargs.pop('myarg') + self.args = args + + def _assert(self): + assert self.args + assert self.myarg + + +class ZippedStoreShaWriter(Sha1Writer): + """Remembers everything someone writes to it""" + __slots__ = ('buf', 'zip') + def __init__(self): + Sha1Writer.__init__(self) + self.buf = StringIO() + self.zip = zlib.compressobj(1) # fastest + + def __getattr__(self, attr): + return getattr(self.buf, attr) + + def write(self, data): + alen = Sha1Writer.write(self, data) + self.buf.write(self.zip.compress(data)) + return alen + + def close(self): + self.buf.write(self.zip.flush()) + + +#} END stream utilitiess + diff --git a/test/test_db.py b/test/test_db.py new file mode 100644 index 000000000..35ba86802 --- /dev/null +++ b/test/test_db.py @@ -0,0 +1,90 @@ +"""Test for object db""" +from test.testlib import * +from lib import ZippedStoreShaWriter + +from git.odb import * +from git.odb.stream import Sha1Writer +from git import Blob +from git.errors import BadObject + + +from cStringIO import StringIO +import os + +class TestDB(TestBase): + """Test the different db class implementations""" + + # data + two_lines = "1234\nhello world" + + all_data = (two_lines, ) + + def _assert_object_writing(self, db): + """General tests to verify object writing, compatible to ObjectDBW + :note: requires write access to the database""" + # start in 'dry-run' mode, using a simple sha1 writer + ostreams = (ZippedStoreShaWriter, None) + for ostreamcls in ostreams: + for data in self.all_data: + dry_run = ostreamcls is not None + ostream = None + if ostreamcls is not None: + ostream = ostreamcls() + assert isinstance(ostream, Sha1Writer) + # END create ostream + + prev_ostream = db.set_ostream(ostream) + assert type(prev_ostream) in ostreams or prev_ostream in ostreams + + istream = IStream(Blob.type, len(data), StringIO(data)) + + # store returns same istream instance, with new sha set + my_istream = db.store(istream) + sha = istream.sha + assert my_istream is istream + assert db.has_object(sha) != dry_run + assert len(sha) == 40 # for now we require 40 byte shas as default + + # verify data - the slow way, we want to run code + if not dry_run: + info = db.info(sha) + assert Blob.type == info.type + assert info.size == len(data) + + ostream = db.stream(sha) + assert ostream.read() == data + assert ostream.type == Blob.type + assert ostream.size == len(data) + else: + self.failUnlessRaises(BadObject, db.info, sha) + self.failUnlessRaises(BadObject, db.stream, sha) + + # DIRECT STREAM COPY + # our data hase been written in object format to the StringIO + # we pasesd as output stream. No physical database representation + # was created. + # Test direct stream copy of object streams, the result must be + # identical to what we fed in + ostream.seek(0) + istream.stream = ostream + assert istream.sha is not None + prev_sha = istream.sha + + db.set_ostream(ZippedStoreShaWriter()) + db.store(istream) + assert istream.sha == prev_sha + new_ostream = db.ostream() + + # note: only works as long our store write uses the same compression + # level, which is zip + assert ostream.getvalue() == new_ostream.getvalue() + # END for each data set + # END for each dry_run mode + + @with_bare_rw_repo + def test_writing(self, rwrepo): + ldb = LooseObjectDB(os.path.join(rwrepo.git_dir, 'objects')) + + # write data + self._assert_object_writing(ldb) + diff --git a/test/test_stream.py b/test/test_stream.py new file mode 100644 index 000000000..020fe6bd3 --- /dev/null +++ b/test/test_stream.py @@ -0,0 +1,172 @@ +"""Test for object db""" +from test.testlib import * +from lib import ( + DummyStream, + DeriveTest, + Sha1Writer + ) + +from git.odb import * +from git import Blob +from cStringIO import StringIO +import tempfile +import os +import zlib + + + + +class TestStream(TestBase): + """Test stream classes""" + + data_sizes = (15, 10000, 1000*1024+512) + + def test_streams(self): + # test info + sha = Blob.NULL_HEX_SHA + s = 20 + info = OInfo(sha, Blob.type, s) + assert info.sha == sha + assert info.type == Blob.type + assert info.size == s + + # test ostream + stream = DummyStream() + ostream = OStream(*(info + (stream, ))) + ostream.read(15) + stream._assert() + assert stream.bytes == 15 + ostream.read(20) + assert stream.bytes == 20 + + # derive with own args + DeriveTest(sha, Blob.type, s, stream, 'mine',myarg = 3)._assert() + + # test istream + istream = IStream(Blob.type, s, stream) + assert istream.sha == None + istream.sha = sha + assert istream.sha == sha + + assert len(istream.binsha) == 20 + assert len(istream.hexsha) == 40 + + assert istream.size == s + istream.size = s * 2 + istream.size == s * 2 + assert istream.type == Blob.type + istream.type = "something" + assert istream.type == "something" + assert istream.stream is stream + istream.stream = None + assert istream.stream is None + + assert istream.error is None + istream.error = Exception() + assert isinstance(istream.error, Exception) + + def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): + """Make stream tests - the orig_stream is seekable, allowing it to be + rewound and reused + :param cdata: the data we expect to read from stream, the contents + :param rewind_stream: function called to rewind the stream to make it ready + for reuse""" + ns = 10 + assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) + + # read in small steps + ss = len(cdata) / ns + for i in range(ns): + data = stream.read(ss) + chunk = cdata[i*ss:(i+1)*ss] + assert data == chunk + # END for each step + rest = stream.read() + if rest: + assert rest == cdata[-len(rest):] + # END handle rest + + rewind_stream(stream) + + # read everything + rdata = stream.read() + assert rdata == cdata + + def test_decompress_reader(self): + for close_on_deletion in range(2): + for with_size in range(2): + for ds in self.data_sizes: + cdata = make_bytes(ds, randomize=False) + + # zdata = zipped actual data + # cdata = original content data + + # create reader + if with_size: + # need object data + zdata = zlib.compress(make_object(Blob.type, cdata)) + type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) + assert size == len(cdata) + assert type == Blob.type + else: + # here we need content data + zdata = zlib.compress(cdata) + reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) + assert reader._s == len(cdata) + # END get reader + + def rewind(r): + r._zip = zlib.decompressobj() + r._br = r._cws = r._cwe = 0 + if with_size: + r._parse_header_info() + # END skip header + # END make rewind func + + self._assert_stream_reader(reader, cdata, rewind) + + # put in a dummy stream for closing + dummy = DummyStream() + reader._m = dummy + + assert not dummy.closed + del(reader) + assert dummy.closed == close_on_deletion + #zdi# + # END for each datasize + # END whether size should be used + # END whether stream should be closed when deleted + + def test_sha_writer(self): + writer = Sha1Writer() + assert 2 == writer.write("hi") + assert len(writer.sha(as_hex=1)) == 40 + assert len(writer.sha(as_hex=0)) == 20 + + # make sure it does something ;) + prev_sha = writer.sha() + writer.write("hi again") + assert writer.sha() != prev_sha + + def test_compressed_writer(self): + for ds in self.data_sizes: + fd, path = tempfile.mkstemp() + ostream = FDCompressedSha1Writer(fd) + data = make_bytes(ds, randomize=False) + + # for now, just a single write, code doesn't care about chunking + assert len(data) == ostream.write(data) + ostream.close() + # its closed already + self.failUnlessRaises(OSError, os.close, fd) + + # read everything back, compare to data we zip + fd = os.open(path, os.O_RDONLY) + written_data = os.read(fd, os.path.getsize(path)) + os.close(fd) + assert written_data == zlib.compress(data, 1) # best speed + + os.remove(path) + # END for each os + + diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 000000000..34572b37e --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,15 @@ +"""Test for object db""" +from test.testlib import * +from git import Blob +from git.odb.utils import ( + to_hex_sha, + to_bin_sha + ) + + +class TestUtils(TestBase): + def test_basics(self): + assert to_hex_sha(Blob.NULL_HEX_SHA) == Blob.NULL_HEX_SHA + assert len(to_bin_sha(Blob.NULL_HEX_SHA)) == 20 + assert to_hex_sha(to_bin_sha(Blob.NULL_HEX_SHA)) == Blob.NULL_HEX_SHA + diff --git a/utils.py b/utils.py new file mode 100644 index 000000000..6863e97b9 --- /dev/null +++ b/utils.py @@ -0,0 +1,38 @@ +import binascii +import os +import errno + +#{ Routines + +hex_to_bin = binascii.a2b_hex +bin_to_hex = binascii.b2a_hex + +def to_hex_sha(sha): + """:return: hexified version of sha""" + if len(sha) == 40: + return sha + return bin_to_hex(sha) + +def to_bin_sha(sha): + if len(sha) == 20: + return sha + return hex_to_bin(sha) + +# errors +ENOENT = errno.ENOENT + +# os shortcuts +exists = os.path.exists +mkdir = os.mkdir +isdir = os.path.isdir +rename = os.rename +dirname = os.path.dirname +join = os.path.join +read = os.read +write = os.write +close = os.close + + +#} END Routines + + From 94c2167bb08c65d4354941e14b1899449efb04f7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 16:14:00 +0200 Subject: [PATCH 0002/1392] Adjusted imports and tests to deal with new folder structure --- db.py | 5 +- exc.py | 14 +++++ fun.py | 2 +- stream.py | 6 +-- test/lib.py | 67 +++++++++++++++++++++++- test/performance/lib.py | 49 ++++++++++++++++++ test/performance/test_db.py | 15 ++++++ test/performance/test_stream.py | 91 +++++++++++++++++++++++++++++++++ test/test_db.py | 28 +++++----- test/test_stream.py | 33 +++++++----- test/test_util.py | 15 ++++++ test/test_utils.py | 15 ------ typ.py | 10 ++++ util.py | 75 +++++++++++++++++++++++++++ utils.py | 38 -------------- 15 files changed, 376 insertions(+), 87 deletions(-) create mode 100644 exc.py create mode 100644 test/performance/lib.py create mode 100644 test/performance/test_db.py create mode 100644 test/performance/test_stream.py create mode 100644 test/test_util.py delete mode 100644 test/test_utils.py create mode 100644 typ.py create mode 100644 util.py delete mode 100644 utils.py diff --git a/db.py b/db.py index 5d3cc6a3f..7ec8a24b3 100644 --- a/db.py +++ b/db.py @@ -1,6 +1,5 @@ """Contains implementations of database retrieveing objects""" -from git.utils import IndexFileSHA1Writer -from git.errors import ( +from exc import ( InvalidDBRoot, BadObject, BadObjectType @@ -14,7 +13,7 @@ OInfo ) -from utils import ( +from util import ( ENOENT, to_hex_sha, exists, diff --git a/exc.py b/exc.py new file mode 100644 index 000000000..3eaf5777d --- /dev/null +++ b/exc.py @@ -0,0 +1,14 @@ +"""Module with common exceptions""" + +class ODBError(Exception): + """All errors thrown by the object database""" + +class InvalidDBRoot(ODBError): + """Thrown if an object database cannot be initialized at the given path""" + +class BadObject(ODBError): + """The object with the given SHA does not exist""" + +class BadObjectType(ODBError): + """The object had an unsupported type""" + diff --git a/fun.py b/fun.py index 3321a8ea4..80b0f41b6 100644 --- a/fun.py +++ b/fun.py @@ -2,7 +2,7 @@ Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" -from git.errors import ( +from exc import ( BadObjectType ) diff --git a/stream.py b/stream.py index da97cf5b6..309df28c4 100644 --- a/stream.py +++ b/stream.py @@ -1,11 +1,11 @@ import zlib from cStringIO import StringIO -from git.utils import make_sha import errno -from utils import ( +from util import ( to_hex_sha, - to_bin_sha, + to_bin_sha, + make_sha, write, close ) diff --git a/test/lib.py b/test/lib.py index d51997488..071e38a5f 100644 --- a/test/lib.py +++ b/test/lib.py @@ -1,12 +1,75 @@ """Utilities used in ODB testing""" -from git.odb import ( +from gitdb import ( OStream, ) -from git.odb.stream import Sha1Writer +from gitdb.stream import Sha1Writer +import sys import zlib +import random +from array import array from cStringIO import StringIO +import unittest +import tempfile +import shutil +import os + + +#{ Bases + +class TestBase(unittest.TestCase): + """Base class for all tests""" + + +#} END bases + +#{ Decorators + +def with_rw_directory(func): + """Create a temporary directory which can be written to, remove it if the + test suceeds, but leave it otherwise to aid additional debugging""" + def wrapper(self): + path = tempfile.mktemp(suffix=func.__name__) + os.mkdir(path) + try: + return func(self, path) + except Exception: + print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + raise + else: + shutil.rmtree(path) + # END handle exception + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper + + +#} END decorators + +#{ Routines + +def make_bytes(size_in_bytes, randomize=False): + """:return: string with given size in bytes + :param randomize: try to produce a very random stream""" + actual_size = size_in_bytes / 4 + producer = xrange(actual_size) + if randomize: + producer = list(producer) + random.shuffle(producer) + # END randomize + a = array('i', producer) + return a.tostring() + + +def make_object(type, data): + """:return: bytes resembling an uncompressed object""" + odata = "blob %i\0" % len(data) + return odata + data + +#} END routines + #{ Stream Utilities class DummyStream(object): diff --git a/test/performance/lib.py b/test/performance/lib.py new file mode 100644 index 000000000..03788c081 --- /dev/null +++ b/test/performance/lib.py @@ -0,0 +1,49 @@ +"""Contains library functions""" +import os +from gitdb.test.lib import * +import shutil +import tempfile + + +#{ Invvariants +k_env_git_repo = "GITDB_TEST_GIT_REPO_BASE" +#} END invariants + + +#{ Utilities +def resolve_or_fail(env_var): + """:return: resolved environment variable or raise EnvironmentError""" + try: + return os.environ[env_var] + except KeyError: + raise EnvironmentError("Please set the %r envrionment variable and retry" % env_var) + # END exception handling + +#} END utilities + + +#{ Base Classes + +class TestBigRepoR(TestBase): + """TestCase providing access to readonly 'big' repositories using the following + member variables: + + * gitrepopath + + * read-only base path of the git source repository, i.e. .../git/.git""" + + #{ Invariants + head_sha_2k = '235d521da60e4699e5bd59ac658b5b48bd76ddca' + head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' + #} END invariants + + @classmethod + def setUpAll(cls): + try: + super(TestBigRepoR, cls).setUpAll() + except AttributeError: + pass + cls.gitrepopath = resolve_or_fail(k_env_git_repo) + + +#} END base classes diff --git a/test/performance/test_db.py b/test/performance/test_db.py new file mode 100644 index 000000000..cd231b650 --- /dev/null +++ b/test/performance/test_db.py @@ -0,0 +1,15 @@ +"""Performance tests for object store""" + +import sys +from time import time + +from lib import ( + TestBigRepoR + ) + +class TestGitDBPerformance(TestBigRepoR): + + def test_random_access(self): + pass + # TODO: use the actual db for this + diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py new file mode 100644 index 000000000..2880c9b79 --- /dev/null +++ b/test/performance/test_stream.py @@ -0,0 +1,91 @@ +"""Performance data streaming performance""" + +from lib import TestBigRepoR +from gitdb.db import * +from gitdb.stream import * + +from cStringIO import StringIO +from time import time +import os +import sys +import stat +import subprocess + + +from lib import ( + TestBigRepoR, + make_bytes, + with_rw_directory + ) + + +def make_memory_file(size_in_bytes, randomize=False): + """:return: tuple(size_of_stream, stream) + :param randomize: try to produce a very random stream""" + d = make_bytes(size_in_bytes, randomize) + return len(d), StringIO(d) + + +class TestObjDBPerformance(TestBigRepoR): + + large_data_size_bytes = 1000*1000*10 # some MiB should do it + moderate_data_size_bytes = 1000*1000*1 # just 1 MiB + + @with_rw_directory + def test_large_data_streaming(self, path): + ldb = LooseObjectDB(path) + + for randomize in range(2): + desc = (randomize and 'random ') or '' + print >> sys.stderr, "Creating %s data ..." % desc + st = time() + size, stream = make_memory_file(self.large_data_size_bytes, randomize) + elapsed = time() - st + print >> sys.stderr, "Done (in %f s)" % elapsed + + # writing - due to the compression it will seem faster than it is + st = time() + sha = ldb.store(IStream('blob', size, stream)).sha + elapsed_add = time() - st + assert ldb.has_object(sha) + db_file = ldb.readable_db_object_path(sha) + fsize_kib = os.path.getsize(db_file) / 1000 + + + size_kib = size / 1000 + print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) + + # reading all at once + st = time() + ostream = ldb.stream(sha) + shadata = ostream.read() + elapsed_readall = time() - st + + stream.seek(0) + assert shadata == stream.getvalue() + print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) + + + # reading in chunks of 1 MiB + cs = 512*1000 + chunks = list() + st = time() + ostream = ldb.stream(sha) + while True: + data = ostream.read(cs) + chunks.append(data) + if len(data) < cs: + break + # END read in chunks + elapsed_readchunks = time() - st + + stream.seek(0) + assert ''.join(chunks) == stream.getvalue() + + cs_kib = cs / 1000 + print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) + + # del db file so git has something to do + os.remove(db_file) + + # END for each randomization factor diff --git a/test/test_db.py b/test/test_db.py index 35ba86802..7f58f4f00 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -1,12 +1,14 @@ """Test for object db""" -from test.testlib import * -from lib import ZippedStoreShaWriter - -from git.odb import * -from git.odb.stream import Sha1Writer -from git import Blob -from git.errors import BadObject +from lib import ( + with_rw_directory, + ZippedStoreShaWriter, + TestBase + ) +from gitdb import * +from gitdb.stream import Sha1Writer +from gitdb.exc import BadObject +from gitdb.typ import str_blob_type from cStringIO import StringIO import os @@ -36,7 +38,7 @@ def _assert_object_writing(self, db): prev_ostream = db.set_ostream(ostream) assert type(prev_ostream) in ostreams or prev_ostream in ostreams - istream = IStream(Blob.type, len(data), StringIO(data)) + istream = IStream(str_blob_type, len(data), StringIO(data)) # store returns same istream instance, with new sha set my_istream = db.store(istream) @@ -48,12 +50,12 @@ def _assert_object_writing(self, db): # verify data - the slow way, we want to run code if not dry_run: info = db.info(sha) - assert Blob.type == info.type + assert str_blob_type == info.type assert info.size == len(data) ostream = db.stream(sha) assert ostream.read() == data - assert ostream.type == Blob.type + assert ostream.type == str_blob_type assert ostream.size == len(data) else: self.failUnlessRaises(BadObject, db.info, sha) @@ -81,9 +83,9 @@ def _assert_object_writing(self, db): # END for each data set # END for each dry_run mode - @with_bare_rw_repo - def test_writing(self, rwrepo): - ldb = LooseObjectDB(os.path.join(rwrepo.git_dir, 'objects')) + @with_rw_directory + def test_writing(self, path): + ldb = LooseObjectDB(path) # write data self._assert_object_writing(ldb) diff --git a/test/test_stream.py b/test/test_stream.py index 020fe6bd3..af7fdc35a 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -1,13 +1,22 @@ """Test for object db""" -from test.testlib import * from lib import ( + TestBase, DummyStream, DeriveTest, - Sha1Writer + Sha1Writer, + make_bytes, + make_object + ) + +from gitdb import * +from gitdb.util import ( + NULL_HEX_SHA + ) + +from gitdb.typ import ( + str_blob_type ) -from git.odb import * -from git import Blob from cStringIO import StringIO import tempfile import os @@ -23,11 +32,11 @@ class TestStream(TestBase): def test_streams(self): # test info - sha = Blob.NULL_HEX_SHA + sha = NULL_HEX_SHA s = 20 - info = OInfo(sha, Blob.type, s) + info = OInfo(sha, str_blob_type, s) assert info.sha == sha - assert info.type == Blob.type + assert info.type == str_blob_type assert info.size == s # test ostream @@ -40,10 +49,10 @@ def test_streams(self): assert stream.bytes == 20 # derive with own args - DeriveTest(sha, Blob.type, s, stream, 'mine',myarg = 3)._assert() + DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() # test istream - istream = IStream(Blob.type, s, stream) + istream = IStream(str_blob_type, s, stream) assert istream.sha == None istream.sha = sha assert istream.sha == sha @@ -54,7 +63,7 @@ def test_streams(self): assert istream.size == s istream.size = s * 2 istream.size == s * 2 - assert istream.type == Blob.type + assert istream.type == str_blob_type istream.type = "something" assert istream.type == "something" assert istream.stream is stream @@ -104,10 +113,10 @@ def test_decompress_reader(self): # create reader if with_size: # need object data - zdata = zlib.compress(make_object(Blob.type, cdata)) + zdata = zlib.compress(make_object(str_blob_type, cdata)) type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) assert size == len(cdata) - assert type == Blob.type + assert type == str_blob_type else: # here we need content data zdata = zlib.compress(cdata) diff --git a/test/test_util.py b/test/test_util.py new file mode 100644 index 000000000..5aac5b84b --- /dev/null +++ b/test/test_util.py @@ -0,0 +1,15 @@ +"""Test for object db""" +from lib import TestBase +from gitdb.util import ( + to_hex_sha, + to_bin_sha, + NULL_HEX_SHA + ) + + +class TestUtils(TestBase): + def test_basics(self): + assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA + assert len(to_bin_sha(NULL_HEX_SHA)) == 20 + assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA + diff --git a/test/test_utils.py b/test/test_utils.py deleted file mode 100644 index 34572b37e..000000000 --- a/test/test_utils.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Test for object db""" -from test.testlib import * -from git import Blob -from git.odb.utils import ( - to_hex_sha, - to_bin_sha - ) - - -class TestUtils(TestBase): - def test_basics(self): - assert to_hex_sha(Blob.NULL_HEX_SHA) == Blob.NULL_HEX_SHA - assert len(to_bin_sha(Blob.NULL_HEX_SHA)) == 20 - assert to_hex_sha(to_bin_sha(Blob.NULL_HEX_SHA)) == Blob.NULL_HEX_SHA - diff --git a/typ.py b/typ.py new file mode 100644 index 000000000..54a1f84be --- /dev/null +++ b/typ.py @@ -0,0 +1,10 @@ +"""Module containing information about types known to the database""" + +#{ String types + +str_blob_type = "blob" +str_commit_type = "commit" +str_tree_type = "tree" +str_tag_type = "tag" + +#} END string types diff --git a/util.py b/util.py new file mode 100644 index 000000000..a6f726399 --- /dev/null +++ b/util.py @@ -0,0 +1,75 @@ +import binascii +import os +import errno + +try: + import hashlib +except ImportError: + import sha + + +#{ Aliases + +hex_to_bin = binascii.a2b_hex +bin_to_hex = binascii.b2a_hex + +# errors +ENOENT = errno.ENOENT + +# os shortcuts +exists = os.path.exists +mkdir = os.mkdir +isdir = os.path.isdir +rename = os.rename +dirname = os.path.dirname +join = os.path.join +read = os.read +write = os.write +close = os.close + +# constants +NULL_HEX_SHA = "0"*40 + +#} END Aliases + + +#{ Routines + +def make_sha(source=''): + """A python2.4 workaround for the sha/hashlib module fiasco + :note: From the dulwich project """ + try: + return hashlib.sha1(source) + except NameError: + sha1 = sha.sha(source) + return sha1 + +def stream_copy(source, destination, chunk_size=512*1024): + """Copy all data from the source stream into the destination stream in chunks + of size chunk_size + + :return: amount of bytes written""" + br = 0 + while True: + chunk = source.read(chunk_size) + destination.write(chunk) + br += len(chunk) + if len(chunk) < chunk_size: + break + # END reading output stream + return br + +def to_hex_sha(sha): + """:return: hexified version of sha""" + if len(sha) == 40: + return sha + return bin_to_hex(sha) + +def to_bin_sha(sha): + if len(sha) == 20: + return sha + return hex_to_bin(sha) + + +#} END routines + diff --git a/utils.py b/utils.py deleted file mode 100644 index 6863e97b9..000000000 --- a/utils.py +++ /dev/null @@ -1,38 +0,0 @@ -import binascii -import os -import errno - -#{ Routines - -hex_to_bin = binascii.a2b_hex -bin_to_hex = binascii.b2a_hex - -def to_hex_sha(sha): - """:return: hexified version of sha""" - if len(sha) == 40: - return sha - return bin_to_hex(sha) - -def to_bin_sha(sha): - if len(sha) == 20: - return sha - return hex_to_bin(sha) - -# errors -ENOENT = errno.ENOENT - -# os shortcuts -exists = os.path.exists -mkdir = os.mkdir -isdir = os.path.isdir -rename = os.rename -dirname = os.path.dirname -join = os.path.join -read = os.read -write = os.write -close = os.close - - -#} END Routines - - From 93f7316425128a498e0581eca12ef7b44380a1ab Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 17:18:34 +0200 Subject: [PATCH 0003/1392] Added async as submodule tests: minimal reorganzation of code --- .gitmodules | 3 +++ ext/async | 1 + test/lib.py | 7 ++++++- test/performance/test_stream.py | 9 +-------- 4 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 .gitmodules create mode 160000 ext/async diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..45ddc0b4c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ext/async"] + path = ext/async + url = git://gitorious.org/git-python/async.git diff --git a/ext/async b/ext/async new file mode 160000 index 000000000..5a13dc577 --- /dev/null +++ b/ext/async @@ -0,0 +1 @@ +Subproject commit 5a13dc5772ec3b00b75c8e3b533051cfb82c4929 diff --git a/test/lib.py b/test/lib.py index 071e38a5f..f0c4064ab 100644 --- a/test/lib.py +++ b/test/lib.py @@ -62,11 +62,16 @@ def make_bytes(size_in_bytes, randomize=False): a = array('i', producer) return a.tostring() - def make_object(type, data): """:return: bytes resembling an uncompressed object""" odata = "blob %i\0" % len(data) return odata + data + +def make_memory_file(size_in_bytes, randomize=False): + """:return: tuple(size_of_stream, stream) + :param randomize: try to produce a very random stream""" + d = make_bytes(size_in_bytes, randomize) + return len(d), StringIO(d) #} END routines diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 2880c9b79..8916d3e58 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -14,18 +14,11 @@ from lib import ( TestBigRepoR, - make_bytes, + make_memory_file, with_rw_directory ) -def make_memory_file(size_in_bytes, randomize=False): - """:return: tuple(size_of_stream, stream) - :param randomize: try to produce a very random stream""" - d = make_bytes(size_in_bytes, randomize) - return len(d), StringIO(d) - - class TestObjDBPerformance(TestBigRepoR): large_data_size_bytes = 1000*1000*10 # some MiB should do it From 10fef8f8e4ee83cf54feadbb5ffb522efec739fc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 17:18:51 +0200 Subject: [PATCH 0004/1392] Added project information --- AUTHORS | 1 + README | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 AUTHORS create mode 100644 README diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 000000000..490baad8e --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Creator: Sebastian Thiel diff --git a/README b/README new file mode 100644 index 000000000..a52d9f508 --- /dev/null +++ b/README @@ -0,0 +1,41 @@ +GtDB +===== + +GitDB allows you to access bare git repositories for reading and writing. It +aims at allowing full access to loose objects as well as packs with performance +and scalability in mind. It operates exclusively on streams, allowing to operate +on large objects with a small memory footprint. + +REQUIREMENTS +============ + +* Python Nose - for running the tests + +SOURCE +====== +The source is available in a git repository at gitorious and github: + +git://gitorious.org/git-python/gitdb.git +git://github.com/Byron/gitdb.git + +Once the clone is complete, please be sure to initialize the submodules using + + cd gitdb + git submodule update --init + +Run the tests with + + nosetests + +MAILING LIST +============ +http://groups.google.com/group/git-python + +ISSUE TRACKER +============= +http://byronimo.lighthouseapp.com/projects/51787-gitpython + +LICENSE +======= + +New BSD License From c64a9741a648526a3d24780ccd5ca193f48684c5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 20:36:00 +0200 Subject: [PATCH 0005/1392] Implemented all async methods, including test which shows how to chain the async method together --- __init__.py | 12 +++++++ db.py | 55 +++++++++++++++++++++--------- ext/async | 2 +- test/__init__.py | 11 ++++++ test/lib.py | 2 +- test/test_db.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++- util.py | 10 ++++++ 7 files changed, 161 insertions(+), 19 deletions(-) diff --git a/__init__.py b/__init__.py index 5789d7eb7..8b0e47b19 100644 --- a/__init__.py +++ b/__init__.py @@ -1,5 +1,17 @@ """Initialize the object database module""" +import sys +import os + +#{ Initialization +def _init_externals(): + """Initialize external projects by putting them into the path""" + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext')) + +#} END initialization + +_init_externals() + # default imports from db import * from stream import * diff --git a/db.py b/db.py index 7ec8a24b3..8107fee25 100644 --- a/db.py +++ b/db.py @@ -14,6 +14,7 @@ ) from util import ( + pool, ENOENT, to_hex_sha, exists, @@ -32,6 +33,11 @@ stream_copy ) + +from async import ( + ChannelThreadTask + ) + import tempfile import mmap import os @@ -40,6 +46,7 @@ __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'LooseObjectDB', 'PackedDB', 'CompoundDB', 'ReferenceDB', 'GitObjectDB' ) + class ObjectDBR(object): """Defines an interface for object database lookup. Objects are identified either by hex-sha (40 bytes) or @@ -52,21 +59,30 @@ def __contains__(self, sha): def has_object(self, sha): """ :return: True if the object identified by the given 40 byte hexsha or 20 bytes - binary sha is contained in the database - :raise BadObject:""" + binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") + def has_object_async(self, reader): + """Return a reader yielding information about the membership of objects + as identified by shas + :param reader: Reader yielding 20 byte or 40 byte shas. + :return: async.Reader yielding tuples of (sha, bool) pairs which indicate + whether the given sha exists in the database or not""" + task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) + return pool.add_task(task) + def info(self, sha): """ :return: OInfo instance :param sha: 40 bytes hexsha or 20 bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - def info_async(self, input_channel): + def info_async(self, reader): """Retrieve information of a multitude of objects asynchronously - :param input_channel: Channel yielding the sha's of the objects of interest - :return: Channel yielding OInfo|InvalidOInfo, in any order""" - raise NotImplementedError("To be implemented in subclass") + :param reader: Channel yielding the sha's of the objects of interest + :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" + task = ChannelThreadTask(reader, str(self.info_async), self.info) + return pool.add_task(task) def stream(self, sha): """:return: OStream instance @@ -74,12 +90,17 @@ def stream(self, sha): :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - def stream_async(self, input_channel): + def stream_async(self, reader): """Retrieve the OStream of multiple objects - :param input_channel: see ``info`` + :param reader: see ``info`` :param max_threads: see ``ObjectDBW.store`` - :return: Channel yielding OStream|InvalidOStream instances in any order""" - raise NotImplementedError("To be implemented in subclass") + :return: async.Reader yielding OStream|InvalidOStream instances in any order + :note: depending on the system configuration, it might not be possible to + read all OStreams at once. Instead, read them individually using reader.read(x) + where x is small enough.""" + # base implementation just uses the stream method repeatedly + task = ChannelThreadTask(reader, str(self.stream_async), self.stream) + return pool.add_task(task) #} END query interface @@ -114,7 +135,7 @@ def store(self, istream): :raise IOError: if data could not be written""" raise NotImplementedError("To be implemented in subclass") - def store_async(self, input_channel): + def store_async(self, reader): """Create multiple new objects in the database asynchronously. The method will return right away, returning an output channel which receives the results as they are computed. @@ -122,13 +143,15 @@ def store_async(self, input_channel): :return: Channel yielding your IStream which served as input, in any order. The IStreams sha will be set to the sha it received during the process, or its error attribute will be set to the exception informing about the error. - :param input_channel: Channel yielding IStream instance. - As the same instances will be used in the output channel, you can create a map - between the id(istream) -> istream - :note:As some ODB implementations implement this operation as atomic, they might + :param reader: async.Reader yielding IStream instances. + The same instances will be used in the output channel as were received + in by the Reader. + :note:As some ODB implementations implement this operation atomic, they might abort the whole operation if one item could not be processed. Hence check how many items have actually been produced.""" - raise NotImplementedError("To be implemented in subclass") + # base implementation uses store to perform the work + task = ChannelThreadTask(reader, str(self.store_async), self.store) + return pool.add_task(task) #} END edit interface diff --git a/ext/async b/ext/async index 5a13dc577..164bb702e 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 5a13dc5772ec3b00b75c8e3b533051cfb82c4929 +Subproject commit 164bb702e3871ab30341a714ef517fb58cd76772 diff --git a/test/__init__.py b/test/__init__.py index 8b1378917..0dec7750f 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1 +1,12 @@ +import gitdb.util + +#{ Initialization +def _init_pool(): + """Assure the pool is actually threaded""" + size = 2 + print "Setting ThreadPool to %i" % size + gitdb.util.pool.set_size(size) + + +#} END initialization diff --git a/test/lib.py b/test/lib.py index f0c4064ab..fc0982bcb 100644 --- a/test/lib.py +++ b/test/lib.py @@ -30,7 +30,7 @@ def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the test suceeds, but leave it otherwise to aid additional debugging""" def wrapper(self): - path = tempfile.mktemp(suffix=func.__name__) + path = tempfile.mktemp(prefix=func.__name__) os.mkdir(path) try: return func(self, path) diff --git a/test/test_db.py b/test/test_db.py index 7f58f4f00..7ba770f48 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -10,6 +10,8 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type +from async import IteratorReader + from cStringIO import StringIO import os @@ -78,10 +80,93 @@ def _assert_object_writing(self, db): new_ostream = db.ostream() # note: only works as long our store write uses the same compression - # level, which is zip + # level, which is zip_best assert ostream.getvalue() == new_ostream.getvalue() # END for each data set # END for each dry_run mode + + def _assert_object_writing_async(self, db): + """Test generic object writing using asynchronous access""" + ni = 5000 + def istream_generator(offset=0, ni=ni): + for data_src in xrange(ni): + data = str(data_src + offset) + yield IStream(str_blob_type, len(data), StringIO(data)) + # END for each item + # END generator utility + + # for now, we are very trusty here as we expect it to work if it worked + # in the single-stream case + + # write objects + reader = IteratorReader(istream_generator()) + istream_reader = db.store_async(reader) + istreams = istream_reader.read() # read all + assert istream_reader.task().error() is None + assert len(istreams) == ni + + for stream in istreams: + assert stream.error is None + assert len(stream.sha) == 40 + assert isinstance(stream, IStream) + # END assert each stream + + # test has-object-async - we must have all previously added ones + reader = IteratorReader( istream.sha for istream in istreams ) + hasobject_reader = db.has_object_async(reader) + count = 0 + for sha, has_object in hasobject_reader: + assert has_object + count += 1 + # END for each sha + assert count == ni + + # read the objects we have just written + reader = IteratorReader( istream.sha for istream in istreams ) + ostream_reader = db.stream_async(reader) + + # read items individually to prevent hitting possible sys-limits + count = 0 + for ostream in ostream_reader: + assert isinstance(ostream, OStream) + count += 1 + # END for each ostream + assert ostream_reader.task().error() is None + assert count == ni + + # get info about our items + reader = IteratorReader( istream.sha for istream in istreams ) + info_reader = db.info_async(reader) + + count = 0 + for oinfo in info_reader: + assert isinstance(oinfo, OInfo) + count += 1 + # END for each oinfo instance + assert count == ni + + + # combined read-write using a converter + # add 2500 items, and obtain their output streams + nni = 2500 + reader = IteratorReader(istream_generator(offset=ni, ni=nni)) + istream_to_sha = lambda istreams: [ istream.sha for istream in istreams ] + + istream_reader = db.store_async(reader) + istream_reader.set_post_cb(istream_to_sha) + + ostream_reader = db.stream_async(istream_reader) + + count = 0 + # read it individually, otherwise we might run into the ulimit + for ostream in ostream_reader: + assert isinstance(ostream, OStream) + count += 1 + # END for each ostream + assert count == nni + + + @with_rw_directory def test_writing(self, path): @@ -89,4 +174,5 @@ def test_writing(self, path): # write data self._assert_object_writing(ldb) + self._assert_object_writing_async(ldb) diff --git a/util.py b/util.py index a6f726399..fd52695bf 100644 --- a/util.py +++ b/util.py @@ -2,11 +2,21 @@ import os import errno +from async import ThreadPool + try: import hashlib except ImportError: import sha +#{ Globals + +# A pool distributing tasks, initially with zero threads, hence everything +# will be handled in the main thread +pool = ThreadPool(0) + +#} END globals + #{ Aliases From 05cee2eb6b35d5216d4dd34bed50cdc921668cd4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 12 Jun 2010 22:22:33 +0200 Subject: [PATCH 0006/1392] Added multi-threading performance tests which show that, during compression and decompression, it is not a tiny bit faster than without, which is due to the GIL and even separately locked zlib module implementations. The only way to make this faster update the resepctive c modules to drop the gil, and their own locks where possible --- ext/async | 2 +- test/performance/test_stream.py | 102 +++++++++++++++++++++++++++++++- 2 files changed, 101 insertions(+), 3 deletions(-) diff --git a/ext/async b/ext/async index 164bb702e..8cfa2542e 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 164bb702e3871ab30341a714ef517fb58cd76772 +Subproject commit 8cfa2542ed623627b5e2e91072368209710e9370 diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 8916d3e58..298207963 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -3,6 +3,14 @@ from lib import TestBigRepoR from gitdb.db import * from gitdb.stream import * +from gitdb.util import pool +from gitdb.typ import str_blob_type +from gitdb.fun import chunk_size + +from async import ( + IteratorReader, + ChannelThreadTask, + ) from cStringIO import StringIO from time import time @@ -19,6 +27,30 @@ ) +#{ Utilities +def read_chunked_stream(stream): + total = 0 + while True: + chunk = stream.read(chunk_size) + total += len(chunk) + if len(chunk) < chunk_size: + break + # END read stream loop + assert total == stream.size + return stream + + +class TestStreamReader(ChannelThreadTask): + """Expects input streams and reads them in chunks. It will read one at a time, + requireing a queue chunk of size 1""" + def __init__(self, *args): + super(TestStreamReader, self).__init__(*args) + self.fun = read_chunked_stream + self.max_chunksize = 1 + + +#} END utilities + class TestObjDBPerformance(TestBigRepoR): large_data_size_bytes = 1000*1000*10 # some MiB should do it @@ -27,7 +59,9 @@ class TestObjDBPerformance(TestBigRepoR): @with_rw_directory def test_large_data_streaming(self, path): ldb = LooseObjectDB(path) + string_ios = list() # list of streams we previously created + # serial mode for randomize in range(2): desc = (randomize and 'random ') or '' print >> sys.stderr, "Creating %s data ..." % desc @@ -35,6 +69,7 @@ def test_large_data_streaming(self, path): size, stream = make_memory_file(self.large_data_size_bytes, randomize) elapsed = time() - st print >> sys.stderr, "Done (in %f s)" % elapsed + string_ios.append(stream) # writing - due to the compression it will seem faster than it is st = time() @@ -78,7 +113,70 @@ def test_large_data_streaming(self, path): cs_kib = cs / 1000 print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) - # del db file so git has something to do + # del db file so we keep something to do os.remove(db_file) - # END for each randomization factor + + + # multi-threaded mode + # want two, should be supported by most of todays cpus + pool.set_size(2) + total_kib = 0 + nsios = len(string_ios) + for stream in string_ios: + stream.seek(0) + total_kib += len(stream.getvalue()) / 1000 + # END rewind + + def istream_iter(): + for stream in string_ios: + stream.seek(0) + yield IStream(str_blob_type, len(stream.getvalue()), stream) + # END for each stream + # END util + + # write multiple objects at once, involving concurrent compression + reader = IteratorReader(istream_iter()) + istream_reader = ldb.store_async(reader) + istream_reader.task().max_chunksize = 1 + + st = time() + istreams = istream_reader.read(nsios) + assert len(istreams) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + + # decompress multiple at once, by reading them + istream_reader = IteratorReader(iter([ i.sha for i in istreams ])) + ostream_reader = ldb.stream_async(istream_reader) + + chunk_task = TestStreamReader(ostream_reader, "chunker", None) + output_reader = pool.add_task(chunk_task) + + st = time() + assert len(output_reader.read(nsios)) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + # store the files, and read them back. For the reading, we use a task + # as well which is chunked into one item per task. Reading all will + # very quickly result in two threads handling two bytestreams of + # chained compression/decompression streams + reader = IteratorReader(istream_iter()) + istream_reader = ldb.store_async(reader) + + istream_to_sha = lambda items: [ i.sha for i in items ] + istream_reader.set_post_cb(istream_to_sha) + + ostream_reader = ldb.stream_async(istream_reader) + chunk_task = TestStreamReader(ostream_reader, "chunker", None) + output_reader = pool.add_task(chunk_task) + + st = time() + assert len(output_reader.read(nsios)) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Compressed and decompressed and read %i KiB of data in loose odb in %f s ( %f Combined KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) From 0ef86550179b9bb9e29ecccdccd586713b9d1752 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 13 Jun 2010 13:44:10 +0200 Subject: [PATCH 0007/1392] Now using the async zlib module if it is available to allow performance gains through multi-threading. --- ext/async | 2 +- fun.py | 2 +- stream.py | 5 +++-- test/lib.py | 2 +- test/performance/test_stream.py | 13 +++++++++---- test/test_stream.py | 2 +- util.py | 6 ++++++ 7 files changed, 22 insertions(+), 10 deletions(-) diff --git a/ext/async b/ext/async index 8cfa2542e..77bf7bef7 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 8cfa2542ed623627b5e2e91072368209710e9370 +Subproject commit 77bf7bef748b019a3a59693cef6d955f74b358ad diff --git a/fun.py b/fun.py index 80b0f41b6..c766f8e09 100644 --- a/fun.py +++ b/fun.py @@ -6,7 +6,7 @@ BadObjectType ) -import zlib +from util import zlib decompressobj = zlib.decompressobj diff --git a/stream.py b/stream.py index 309df28c4..10bc8901a 100644 --- a/stream.py +++ b/stream.py @@ -1,4 +1,4 @@ -import zlib + from cStringIO import StringIO import errno @@ -7,7 +7,8 @@ to_bin_sha, make_sha, write, - close + close, + zlib ) __all__ = ('OInfo', 'OStream', 'IStream', 'InvalidOInfo', 'InvalidOStream', diff --git a/test/lib.py b/test/lib.py index fc0982bcb..723958ca1 100644 --- a/test/lib.py +++ b/test/lib.py @@ -3,9 +3,9 @@ OStream, ) from gitdb.stream import Sha1Writer +from gitdb.util import zlib import sys -import zlib import random from array import array from cStringIO import StringIO diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 298207963..5de463ee2 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -1,5 +1,4 @@ """Performance data streaming performance""" - from lib import TestBigRepoR from gitdb.db import * from gitdb.stream import * @@ -53,7 +52,7 @@ def __init__(self, *args): class TestObjDBPerformance(TestBigRepoR): - large_data_size_bytes = 1000*1000*10 # some MiB should do it + large_data_size_bytes = 1000*1000*50 # some MiB should do it moderate_data_size_bytes = 1000*1000*1 # just 1 MiB @with_rw_directory @@ -147,19 +146,22 @@ def istream_iter(): print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - # decompress multiple at once, by reading them + # chunk size is not important as the stream will not really be decompressed + + # until its read istream_reader = IteratorReader(iter([ i.sha for i in istreams ])) ostream_reader = ldb.stream_async(istream_reader) chunk_task = TestStreamReader(ostream_reader, "chunker", None) output_reader = pool.add_task(chunk_task) + output_reader.task().max_chunksize = 1 st = time() assert len(output_reader.read(nsios)) == nsios elapsed = time() - st - print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Read KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) # store the files, and read them back. For the reading, we use a task # as well which is chunked into one item per task. Reading all will @@ -167,13 +169,16 @@ def istream_iter(): # chained compression/decompression streams reader = IteratorReader(istream_iter()) istream_reader = ldb.store_async(reader) + istream_reader.task().max_chunksize = 1 istream_to_sha = lambda items: [ i.sha for i in items ] istream_reader.set_post_cb(istream_to_sha) ostream_reader = ldb.stream_async(istream_reader) + chunk_task = TestStreamReader(ostream_reader, "chunker", None) output_reader = pool.add_task(chunk_task) + output_reader.max_chunksize = 1 st = time() assert len(output_reader.read(nsios)) == nsios diff --git a/test/test_stream.py b/test/test_stream.py index af7fdc35a..4f022286e 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -13,6 +13,7 @@ NULL_HEX_SHA ) +from gitdb.util import zlib from gitdb.typ import ( str_blob_type ) @@ -20,7 +21,6 @@ from cStringIO import StringIO import tempfile import os -import zlib diff --git a/util.py b/util.py index fd52695bf..6b8862472 100644 --- a/util.py +++ b/util.py @@ -2,6 +2,12 @@ import os import errno +try: + import async.mod.zlib as zlib +except ImportError: + import zlib +# END try async zlib + from async import ThreadPool try: From 97a17dc4f3af188c8c00b0b265f17c26c9c96ddc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 14 Jun 2010 18:16:31 +0200 Subject: [PATCH 0008/1392] Made the db module a package to have enough room for expansion --- db/__init__.py | 7 ++ db.py => db/base.py | 218 +------------------------------------------- db/git.py | 33 +++++++ db/loose.py | 186 +++++++++++++++++++++++++++++++++++++ db/pack.py | 11 +++ db/ref.py | 7 ++ ext/async | 2 +- 7 files changed, 250 insertions(+), 214 deletions(-) create mode 100644 db/__init__.py rename db.py => db/base.py (50%) create mode 100644 db/git.py create mode 100644 db/loose.py create mode 100644 db/pack.py create mode 100644 db/ref.py diff --git a/db/__init__.py b/db/__init__.py new file mode 100644 index 000000000..05d9b21b3 --- /dev/null +++ b/db/__init__.py @@ -0,0 +1,7 @@ + +from base import * +from loose import * +from pack import * +from git import * +from ref import * + diff --git a/db.py b/db/base.py similarity index 50% rename from db.py rename to db/base.py index 8107fee25..2cda0ea0a 100644 --- a/db.py +++ b/db/base.py @@ -1,50 +1,15 @@ """Contains implementations of database retrieveing objects""" -from exc import ( - InvalidDBRoot, - BadObject, - BadObjectType - ) - -from stream import ( - DecompressMemMapReader, - FDCompressedSha1Writer, - Sha1Writer, - OStream, - OInfo - ) - -from util import ( +from gitdb.util import ( pool, - ENOENT, - to_hex_sha, - exists, - hex_to_bin, - isdir, - mkdir, - rename, - dirname, join ) -from fun import ( - chunk_size, - loose_object_header_info, - write_object, - stream_copy - ) - - from async import ( ChannelThreadTask ) -import tempfile -import mmap -import os - -__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'LooseObjectDB', 'PackedDB', - 'CompoundDB', 'ReferenceDB', 'GitObjectDB' ) +__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB') class ObjectDBR(object): @@ -104,6 +69,7 @@ def stream_async(self, reader): #} END query interface + class ObjectDBW(object): """Defines an interface to create objects in the database""" @@ -183,181 +149,7 @@ def db_path(self, rela_path): return join(self._root_path, rela_path) #} END interface - - -class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): - """A database which operates on loose object files""" - - # CONFIGURATION - # chunks in which data will be copied between streams - stream_chunk_size = chunk_size - - - def __init__(self, root_path): - super(LooseObjectDB, self).__init__(root_path) - self._hexsha_to_file = dict() - # Additional Flags - might be set to 0 after the first failure - # Depending on the root, this might work for some mounts, for others not, which - # is why it is per instance - self._fd_open_flags = getattr(os, 'O_NOATIME', 0) - - #{ Interface - def object_path(self, hexsha): - """ - :return: path at which the object with the given hexsha would be stored, - relative to the database root""" - return join(hexsha[:2], hexsha[2:]) - - def readable_db_object_path(self, hexsha): - """ - :return: readable object path to the object identified by hexsha - :raise BadObject: If the object file does not exist""" - try: - return self._hexsha_to_file[hexsha] - except KeyError: - pass - # END ignore cache misses - - # try filesystem - path = self.db_path(self.object_path(hexsha)) - if exists(path): - self._hexsha_to_file[hexsha] = path - return path - # END handle cache - raise BadObject(hexsha) - - #} END interface - - def _map_loose_object(self, sha): - """ - :return: memory map of that file to allow random read access - :raise BadObject: if object could not be located""" - db_path = self.db_path(self.object_path(to_hex_sha(sha))) - try: - fd = os.open(db_path, os.O_RDONLY|self._fd_open_flags) - except OSError,e: - if e.errno != ENOENT: - # try again without noatime - try: - fd = os.open(db_path, os.O_RDONLY) - except OSError: - raise BadObject(to_hex_sha(sha)) - # didn't work because of our flag, don't try it again - self._fd_open_flags = 0 - else: - raise BadObject(to_hex_sha(sha)) - # END handle error - # END exception handling - try: - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - finally: - os.close(fd) - # END assure file is closed - - def set_ostream(self, stream): - """:raise TypeError: if the stream does not support the Sha1Writer interface""" - if stream is not None and not isinstance(stream, Sha1Writer): - raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) - return super(LooseObjectDB, self).set_ostream(stream) - - def info(self, sha): - m = self._map_loose_object(sha) - try: - type, size = loose_object_header_info(m) - return OInfo(sha, type, size) - finally: - m.close() - # END assure release of system resources - - def stream(self, sha): - m = self._map_loose_object(sha) - type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) - return OStream(sha, type, size, stream) - - def has_object(self, sha): - try: - self.readable_db_object_path(to_hex_sha(sha)) - return True - except BadObject: - return False - # END check existance - - def store(self, istream): - """note: The sha we produce will be hex by nature""" - tmp_path = None - writer = self.ostream() - if writer is None: - # open a tmp file to write the data to - fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - writer = FDCompressedSha1Writer(fd) - # END handle custom writer - - try: - try: - if istream.sha is not None: - stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size) - else: - # write object with header, we have to make a new one - write_object(istream.type, istream.size, istream.read, writer.write, - chunk_size=self.stream_chunk_size) - # END handle direct stream copies - except: - if tmp_path: - os.remove(tmp_path) - raise - # END assure tmpfile removal on error - finally: - if tmp_path: - writer.close() - # END assure target stream is closed - - sha = istream.sha or writer.sha(as_hex=True) - - if tmp_path: - obj_path = self.db_path(self.object_path(sha)) - obj_dir = dirname(obj_path) - if not isdir(obj_dir): - mkdir(obj_dir) - # END handle destination directory - rename(tmp_path, obj_path) - # END handle dry_run - - istream.sha = sha - return istream - - -class PackedDB(FileDBBase, ObjectDBR): - """A database operating on a set of object packs""" - - + class CompoundDB(ObjectDBR): """A database which delegates calls to sub-databases""" - - -class ReferenceDB(CompoundDB): - """A database consisting of database referred to in a file""" - - -#class GitObjectDB(CompoundDB, ObjectDBW): -class GitObjectDB(LooseObjectDB): - """A database representing the default git object store, which includes loose - objects, pack files and an alternates file - - It will create objects only in the loose object database. - :note: for now, we use the git command to do all the lookup, just until he - have packs and the other implementations - """ - def __init__(self, root_path, git): - """Initialize this instance with the root and a git command""" - super(GitObjectDB, self).__init__(root_path) - self._git = git - - def info(self, sha): - t = self._git.get_object_header(sha) - return OInfo(*t) - - def stream(self, sha): - """For now, all lookup is done by git itself""" - t = self._git.stream_object_data(sha) - return OStream(*t) - + # TODO diff --git a/db/git.py b/db/git.py new file mode 100644 index 000000000..d2477d7b1 --- /dev/null +++ b/db/git.py @@ -0,0 +1,33 @@ + +from gitdb.stream import ( + OInfo, + OStream + ) + +from loose import LooseObjectDB + +__all__ = ('GitObjectDB', ) + +#class GitObjectDB(CompoundDB, ObjectDBW): +class GitObjectDB(LooseObjectDB): + """A database representing the default git object store, which includes loose + objects, pack files and an alternates file + + It will create objects only in the loose object database. + :note: for now, we use the git command to do all the lookup, just until he + have packs and the other implementations + """ + def __init__(self, root_path, git): + """Initialize this instance with the root and a git command""" + super(GitObjectDB, self).__init__(root_path) + self._git = git + + def info(self, sha): + t = self._git.get_object_header(sha) + return OInfo(*t) + + def stream(self, sha): + """For now, all lookup is done by git itself""" + t = self._git.stream_object_data(sha) + return OStream(*t) + diff --git a/db/loose.py b/db/loose.py new file mode 100644 index 000000000..37aad8c6f --- /dev/null +++ b/db/loose.py @@ -0,0 +1,186 @@ +from base import ( + FileDBBase, + ObjectDBR, + ObjectDBW + ) + + +from gitdb.exc import ( + InvalidDBRoot, + BadObject, + ) + +from gitdb.stream import ( + DecompressMemMapReader, + FDCompressedSha1Writer, + Sha1Writer, + OStream, + OInfo + ) + +from gitdb.util import ( + ENOENT, + to_hex_sha, + exists, + isdir, + mkdir, + rename, + dirname, + join + ) + +from gitdb.fun import ( + chunk_size, + loose_object_header_info, + write_object, + stream_copy + ) + +import tempfile +import mmap +import os + + +__all__ = ( 'LooseObjectDB', ) + + +class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): + """A database which operates on loose object files""" + + # CONFIGURATION + # chunks in which data will be copied between streams + stream_chunk_size = chunk_size + + + def __init__(self, root_path): + super(LooseObjectDB, self).__init__(root_path) + self._hexsha_to_file = dict() + # Additional Flags - might be set to 0 after the first failure + # Depending on the root, this might work for some mounts, for others not, which + # is why it is per instance + self._fd_open_flags = getattr(os, 'O_NOATIME', 0) + + #{ Interface + def object_path(self, hexsha): + """ + :return: path at which the object with the given hexsha would be stored, + relative to the database root""" + return join(hexsha[:2], hexsha[2:]) + + def readable_db_object_path(self, hexsha): + """ + :return: readable object path to the object identified by hexsha + :raise BadObject: If the object file does not exist""" + try: + return self._hexsha_to_file[hexsha] + except KeyError: + pass + # END ignore cache misses + + # try filesystem + path = self.db_path(self.object_path(hexsha)) + if exists(path): + self._hexsha_to_file[hexsha] = path + return path + # END handle cache + raise BadObject(hexsha) + + #} END interface + + def _map_loose_object(self, sha): + """ + :return: memory map of that file to allow random read access + :raise BadObject: if object could not be located""" + db_path = self.db_path(self.object_path(to_hex_sha(sha))) + try: + fd = os.open(db_path, os.O_RDONLY|self._fd_open_flags) + except OSError,e: + if e.errno != ENOENT: + # try again without noatime + try: + fd = os.open(db_path, os.O_RDONLY) + except OSError: + raise BadObject(to_hex_sha(sha)) + # didn't work because of our flag, don't try it again + self._fd_open_flags = 0 + else: + raise BadObject(to_hex_sha(sha)) + # END handle error + # END exception handling + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + finally: + os.close(fd) + # END assure file is closed + + def set_ostream(self, stream): + """:raise TypeError: if the stream does not support the Sha1Writer interface""" + if stream is not None and not isinstance(stream, Sha1Writer): + raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) + return super(LooseObjectDB, self).set_ostream(stream) + + def info(self, sha): + m = self._map_loose_object(sha) + try: + type, size = loose_object_header_info(m) + return OInfo(sha, type, size) + finally: + m.close() + # END assure release of system resources + + def stream(self, sha): + m = self._map_loose_object(sha) + type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) + return OStream(sha, type, size, stream) + + def has_object(self, sha): + try: + self.readable_db_object_path(to_hex_sha(sha)) + return True + except BadObject: + return False + # END check existance + + def store(self, istream): + """note: The sha we produce will be hex by nature""" + tmp_path = None + writer = self.ostream() + if writer is None: + # open a tmp file to write the data to + fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) + writer = FDCompressedSha1Writer(fd) + # END handle custom writer + + try: + try: + if istream.sha is not None: + stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size) + else: + # write object with header, we have to make a new one + write_object(istream.type, istream.size, istream.read, writer.write, + chunk_size=self.stream_chunk_size) + # END handle direct stream copies + except: + if tmp_path: + os.remove(tmp_path) + raise + # END assure tmpfile removal on error + finally: + if tmp_path: + writer.close() + # END assure target stream is closed + + sha = istream.sha or writer.sha(as_hex=True) + + if tmp_path: + obj_path = self.db_path(self.object_path(sha)) + obj_dir = dirname(obj_path) + if not isdir(obj_dir): + mkdir(obj_dir) + # END handle destination directory + rename(tmp_path, obj_path) + # END handle dry_run + + istream.sha = sha + return istream + diff --git a/db/pack.py b/db/pack.py new file mode 100644 index 000000000..e57241a32 --- /dev/null +++ b/db/pack.py @@ -0,0 +1,11 @@ +"""Module containing a database to deal with packs""" +from base import ( + FileDBBase, + ObjectDBR + ) + +__all__ = ('PackedDB', ) + +class PackedDB(FileDBBase, ObjectDBR): + """A database operating on a set of object packs""" + diff --git a/db/ref.py b/db/ref.py new file mode 100644 index 000000000..2c63884bc --- /dev/null +++ b/db/ref.py @@ -0,0 +1,7 @@ +from base import CompoundDB + +__all__ = ('CompoundDB', ) + +class ReferenceDB(CompoundDB): + """A database consisting of database referred to in a file""" + diff --git a/ext/async b/ext/async index 77bf7bef7..af0040b0f 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 77bf7bef748b019a3a59693cef6d955f74b358ad +Subproject commit af0040b0f3c6ede3be5b2d6bc69f6ea5ac53c36c From 133988a9b53400810d2baea9cc817c67dd1577a9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 14 Jun 2010 18:27:29 +0200 Subject: [PATCH 0009/1392] update db test to allow testing of individual database types, giving more fine-grained control over testing the db packge --- test/db/__init__.py | 0 test/{test_db.py => db/lib.py} | 31 ++++++++++++++----------------- test/db/test_loose.py | 13 +++++++++++++ 3 files changed, 27 insertions(+), 17 deletions(-) create mode 100644 test/db/__init__.py rename test/{test_db.py => db/lib.py} (93%) create mode 100644 test/db/test_loose.py diff --git a/test/db/__init__.py b/test/db/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/test_db.py b/test/db/lib.py similarity index 93% rename from test/test_db.py rename to test/db/lib.py index 7ba770f48..8d61e677b 100644 --- a/test/test_db.py +++ b/test/db/lib.py @@ -1,22 +1,29 @@ -"""Test for object db""" -from lib import ( +"""Base classes for object db testing""" +from gitdb.test.lib import ( with_rw_directory, ZippedStoreShaWriter, TestBase ) -from gitdb import * -from gitdb.stream import Sha1Writer +from gitdb.stream import ( + Sha1Writer, + IStream, + OStream, + OInfo + ) + from gitdb.exc import BadObject from gitdb.typ import str_blob_type from async import IteratorReader from cStringIO import StringIO -import os + + +__all__ = ('TestDBBase', 'with_rw_directory' ) -class TestDB(TestBase): - """Test the different db class implementations""" +class TestDBBase(TestBase): + """Base class providing testing routines on databases""" # data two_lines = "1234\nhello world" @@ -166,13 +173,3 @@ def istream_generator(offset=0, ni=ni): assert count == nni - - - @with_rw_directory - def test_writing(self, path): - ldb = LooseObjectDB(path) - - # write data - self._assert_object_writing(ldb) - self._assert_object_writing_async(ldb) - diff --git a/test/db/test_loose.py b/test/db/test_loose.py new file mode 100644 index 000000000..70cd7742c --- /dev/null +++ b/test/db/test_loose.py @@ -0,0 +1,13 @@ +from lib import * +from gitdb.db import LooseObjectDB + +class TestLooseDB(TestDBBase): + + @with_rw_directory + def test_writing(self, path): + ldb = LooseObjectDB(path) + + # write data + self._assert_object_writing(ldb) + self._assert_object_writing_async(ldb) + From 937d592ad08cff4bcb675a96d62534c65cc8015c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 01:06:01 +0200 Subject: [PATCH 0010/1392] Added LockedFD class including test, it moved 'down' from git-python --- test/test_util.py | 78 ++++++++++++++++++++++++- util.py | 144 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 221 insertions(+), 1 deletion(-) diff --git a/test/test_util.py b/test/test_util.py index 5aac5b84b..2272b53e5 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,9 +1,13 @@ """Test for object db""" +import tempfile +import os + from lib import TestBase from gitdb.util import ( to_hex_sha, to_bin_sha, - NULL_HEX_SHA + NULL_HEX_SHA, + LockedFD ) @@ -12,4 +16,76 @@ def test_basics(self): assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA assert len(to_bin_sha(NULL_HEX_SHA)) == 20 assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA + + def _cmp_contents(self, file_path, data): + # raise if data from file at file_path + # does not match data string + fp = open(file_path, "rb") + try: + assert fp.read() == data + finally: + fp.close() + + def test_lockedfd(self): + my_file = tempfile.mktemp() + orig_data = "hello" + new_data = "world" + my_file_fp = open(my_file, "wb") + my_file_fp.write(orig_data) + my_file_fp.close() + + try: + lfd = LockedFD(my_file) + lockfilepath = lfd._lockfilepath() + + # cannot end before it was started + self.failUnlessRaises(AssertionError, lfd.rollback) + self.failUnlessRaises(AssertionError, lfd.commit) + + # open for writing + assert not os.path.isfile(lockfilepath) + wfd = lfd.open(write=True) + assert lfd._fd is wfd + assert os.path.isfile(lockfilepath) + + # write data and fail + os.write(wfd, new_data) + lfd.rollback() + assert lfd._fd is None + self._cmp_contents(my_file, orig_data) + assert not os.path.isfile(lockfilepath) + + # additional call doesnt fail + lfd.commit() + lfd.rollback() + + # test reading + lfd = LockedFD(my_file) + rfd = lfd.open(write=False) + assert os.read(rfd, len(orig_data)) == orig_data + + assert os.path.isfile(lockfilepath) + # deletion rolls back + del(lfd) + assert not os.path.isfile(lockfilepath) + + + # write data - concurrently + lfd = LockedFD(my_file) + olfd = LockedFD(my_file) + assert not os.path.isfile(lockfilepath) + wfdstream = lfd.open(write=True, stream=True) # this time as stream + assert os.path.isfile(lockfilepath) + # another one fails + self.failUnlessRaises(IOError, olfd.open) + + wfdstream.write(new_data) + lfd.commit() + assert not os.path.isfile(lockfilepath) + self._cmp_contents(my_file, new_data) + + # could test automatic _end_writing on destruction + finally: + os.remove(my_file) + # END final cleanup diff --git a/util.py b/util.py index 6b8862472..291855630 100644 --- a/util.py +++ b/util.py @@ -1,5 +1,6 @@ import binascii import os +import sys import errno try: @@ -89,3 +90,146 @@ def to_bin_sha(sha): #} END routines + +#{ Utilities + + +class FDStreamWrapper(object): + """A simple wrapper providing the most basic functions on a file descriptor + with the fileobject interface. Cannot use os.fdopen as the resulting stream + takes ownership""" + __slots__ = ("_fd", '_pos') + def __init__(self, fd): + self._fd = fd + self._pos = 0 + + def write(self, data): + self._pos += len(data) + os.write(self._fd, data) + + def read(self, count=0): + if count == 0: + count = os.path.getsize(self._filepath) + # END handle read everything + + bytes = os.read(self._fd, count) + self._pos += len(bytes) + return bytes + + def fileno(self): + return self._fd + + def tell(self): + return self._pos + + +class LockedFD(object): + """This class facilitates a safe read and write operation to a file on disk. + If we write to 'file', we obtain a lock file at 'file.lock' and write to + that instead. If we succeed, the lock file will be renamed to overwrite + the original file. + + When reading, we obtain a lock file, but to prevent other writers from + succeeding while we are reading the file. + + This type handles error correctly in that it will assure a consistent state + on destruction. + + :note: with this setup, parallel reading is not possible""" + __slots__ = ("_filepath", '_fd', '_write') + + def __init__(self, filepath): + """Initialize an instance with the givne filepath""" + self._filepath = filepath + self._fd = None + self._write = None # if True, we write a file + + def __del__(self): + # will do nothing if the file descriptor is already closed + if self._fd is not None: + self.rollback() + + def _lockfilepath(self): + return "%s.lock" % self._filepath + + def open(self, write=False, stream=False): + """Open the file descriptor for reading or writing, both in binary mode. + :param write: if True, the file descriptor will be opened for writing. Other + wise it will be opened read-only. + :param stream: if True, the file descriptor will be wrapped into a simple stream + object which supports only reading or writing + :return: fd to read from or write to. It is still maintained by this instance + and must not be closed directly + :raise IOError: if the lock could not be retrieved + :raise OSError: If the actual file could not be opened for reading + :note: must only be called once""" + if self._write is not None: + raise AssertionError("Called %s multiple times" % self.open) + + self._write = write + + # try to open the lock file + binary = getattr(os, 'O_BINARY', 0) + lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary + try: + fd = os.open(self._lockfilepath(), lockmode) + if not write: + os.close(fd) + else: + self._fd = fd + # END handle file descriptor + except OSError: + raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) + # END handle lock retrieval + + # open actual file if required + if self._fd is None: + # we could specify exlusive here, as we obtained the lock anyway + self._fd = os.open(self._filepath, os.O_RDONLY | binary) + # END open descriptor for reading + + if stream: + return FDStreamWrapper(self._fd) + else: + return self._fd + # END handle stream + + def commit(self): + """When done writing, call this function to commit your changes into the + actual file. + The file descriptor will be closed, and the lockfile handled. + :note: can be called multiple times""" + self._end_writing(successful=True) + + def rollback(self): + """Abort your operation without any changes. The file descriptor will be + closed, and the lock released. + :note: can be called multiple times""" + self._end_writing(successful=False) + + def _end_writing(self, successful=True): + """Handle the lock according to the write mode """ + if self._write is None: + raise AssertionError("Cannot end operation if it wasn't started yet") + + if self._fd is None: + return + + os.close(self._fd) + self._fd = None + + lockfile = self._lockfilepath() + if self._write and successful: + # on windows, rename does not silently overwrite the existing one + if sys.platform == "win32": + if os.path.isfile(self._filepath): + os.remove(self._filepath) + # END remove if exists + # END win32 special handling + os.rename(lockfile, self._filepath) + else: + # just delete the file so far, we failed + os.remove(lockfile) + # END successful handling + +#} END utilities From f50643ff166180d3a048ff55422d84e631e831b1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 01:07:32 +0200 Subject: [PATCH 0011/1392] Added basic frame for packfile implementation and testing, as well as the testing of the corresponding PackedDB. It wants to be filled out now, but the design not yet done either actually --- db/pack.py | 33 ++++++++++++++++ exc.py | 2 + pack.py | 1 + test/db/lib.py | 4 +- test/db/test_pack.py | 12 ++++++ ...fdfa9e156ab73caae3b6da867192221f2089c2.idx | Bin 0 -> 1912 bytes ...dfa9e156ab73caae3b6da867192221f2089c2.pack | Bin 0 -> 51875 bytes test/lib.py | 37 ++++++++++++++++++ test/test_pack.py | 16 ++++++++ 9 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 pack.py create mode 100644 test/db/test_pack.py create mode 100644 test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx create mode 100644 test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack create mode 100644 test/test_pack.py diff --git a/db/pack.py b/db/pack.py index e57241a32..a850e0fb2 100644 --- a/db/pack.py +++ b/db/pack.py @@ -4,8 +4,41 @@ ObjectDBR ) +from gitdb.exc import ( + UnsupportedOperation, + ) + __all__ = ('PackedDB', ) class PackedDB(FileDBBase, ObjectDBR): """A database operating on a set of object packs""" + def __init__(self, root_path): + super(PackedDB, self).__init__(root_path) + + + #{ Object DB Read + + def has_object(self, sha): + raise NotImplementedError() + + def info(self, sha): + raise NotImplementedError() + + def stream(self, sha): + raise NotImplementedError() + + #} END object db read + + #{ object db write + + def store(self, istream): + """Storing individual objects is not feasible as a pack is designed to + hold multiple objects. Writing or rewriting packs for single objects is + inefficient""" + raise UnsupportedOperation() + + def store_async(self, reader): + raise NotImplementedError() + + #} END object db write diff --git a/exc.py b/exc.py index 3eaf5777d..482726e3b 100644 --- a/exc.py +++ b/exc.py @@ -12,3 +12,5 @@ class BadObject(ODBError): class BadObjectType(ODBError): """The object had an unsupported type""" +class UnsupportedOperation(ODBError): + """Thrown if the given operation cannot be supported by the object database""" diff --git a/pack.py b/pack.py new file mode 100644 index 000000000..676fa26c5 --- /dev/null +++ b/pack.py @@ -0,0 +1 @@ +"""Contains PackIndex and PackFile implementations""" diff --git a/test/db/lib.py b/test/db/lib.py index 8d61e677b..738eeb851 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -1,6 +1,7 @@ """Base classes for object db testing""" from gitdb.test.lib import ( with_rw_directory, + with_packs, ZippedStoreShaWriter, TestBase ) @@ -16,11 +17,10 @@ from gitdb.typ import str_blob_type from async import IteratorReader - from cStringIO import StringIO -__all__ = ('TestDBBase', 'with_rw_directory' ) +__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs' ) class TestDBBase(TestBase): """Base class providing testing routines on databases""" diff --git a/test/db/test_pack.py b/test/db/test_pack.py new file mode 100644 index 000000000..29b348876 --- /dev/null +++ b/test/db/test_pack.py @@ -0,0 +1,12 @@ +from lib import * +from gitdb.db import PackedDB + +class TestPackDB(TestDBBase): + + @with_rw_directory + @with_packs + def test_writing(self, path): + ldb = PackedDB(path) + # TODO + + diff --git a/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx b/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx new file mode 100644 index 0000000000000000000000000000000000000000..fda5969bc69c10781d3159ecfc6a3caf36d3eea9 GIT binary patch literal 1912 zcmexg;-AdGz`z8=t%7h5R~Q zTXcX;mTl)hi_XvCD|Od)tJiK`87aAUf}`>E=V$DUY<600OIVw9x2C82n)#~P2M@Ja zwOIbMP34ktToN=Rxv~G6@XomAwC?=}9jBkO$(>>z`H-WFX~s6!!YzA>qI^uhIo^J| z?vlaQi2geQIj$C4F3k3sz99UGVC2uTnjZyzSJz2M-S_Pckk%}5U!fXnYJ9AH+OGu-(2 zw2i8TzvWXCk`_i)Jiqg$`DTLNnLou=p&UZzJav6maeY%g(W3UP=ufKhMAMBQBe=KE zJ1zE?^+QYJ?e~j*PoA-?_@BJT9S&UL4_P#nPuL6x5;q0p)nH(}Y$K7@A^B4QY zKFoO;LE2$LFBol?K3 zi)vlv$=yb0{+D?ki{h$wQz{2t5R^_?jnqvlONmddNX(5LTPZTs!na$V15Z&z(BqE8Ww(Zh%a!T)|W_!#O7_n1U3oD_JRosc_f1Thc=bDcsLm_cg&T4phZO^m^jPta0JV+A8E zvs$ok?@62+r$yBZ-aJZ?nkyUQ86c_*wSGfbtxfvQ6Xw1C~%USqch}(FGMK33|-WamcqvZ$n$kNsI{a7G*SUdu%VrNl}1Hc#tcwRkVbb{?_%)vi{yF2qOM zdf{c%x}@)?Kdg!$Qg35^ml1fJos3IP!!Qs)_ddm3Akp~SF+x=(7Ercufw4VPvPfJ7 zSAyHOM?h?!-g{#g0SaVW5oM<(`%Ih;Ud3vlRs&(R8Y59gbLCyO0jxn&!02*nQd7>} z*~&4uK#{yEqj$cb7`~0KzQg-%TRObJ8?U@|c)l#-d&qve_R@dm+Semk7rpg_m|(&K zqIuAD84v4UwRt^%f}PVG=3vana@nL!U3DtIPk$JiJ87m=5}F!#oSlrzN&_(%h1cdO ze!3|LQ*)Wziil7U5qyYT{%O;8k|C3UKE8wKLKiOF9nSZi(<&kuBaJm88s;hDpcQ&4 zBWI3AtGxD1nOa7!%g_p@ zm5A%#I@J-wfD*0ZQBdVwAZ}@=QsHgMslWkV-RN?G$75=9!}>fGUv9fl#wTE03MRB< z20E%KC2KpTwp#yMlXt8~1YcbaC)l3D!{}0O-I6C4E?x0w3~lb_R#RwOSW$sbgOGdp z+ujp^mvn>8PYT@^-w<*rdAP1|-jh4Oq-#t0*VitP54%c*{gNJdoSo6nO2jY_2Jm~H zVs5KK*ZkSE%_1TQB7)B#lVq|9v`J~RY#-lxkOe&mo@V&?e#~L&Oa#<(ZjFy79QcdhijCMQ}%?^UqI2&>R{s9L=x^xaohQ{W^; z>m|p7@#v&fWKFNP#pzZH=`A^i=NiilZ|9r}Ji{xFm1J^KV*U!97T8F-wXjloI-F%(7j`HDw31!3BkWRgjXh~P#9KjI~M>5O%fl4d~v z-a+lwh1+}2;k3#OCbHs4)~KWo5_P~REhZPx#DSgm)@YJu5ErVP8+cF34Mt0^$@L}? zp;qJ)C(SkvtL;p12Y=&3Fb*j>XK)CNiUUc~Yfmcd`xxaRxIQ`*w6w8Qcv_yK?Gd=+0p zp?x!v8+e?ZjLk{|F%(Akd5XKM3Zwa-q=*P^MDQ6TxpSuz+ewBr#rpORMHj9Fx8FJ6 zX_W{_DveS}BpzKr8r7)P7}k+1OV$}Fqp?D=B~;1{wD3KWLOXb^dgDoO=K`1BMXOCO zqK?>06Mp<*Wf%;1(P>FmU@rzQxe|OF6_1g)RL*DdW5$q=(5KD|Nk z1-uo-S5#4>Gea;rE>bhmO5PZBP6nlkZPeKkqQrI4j)FL_a~+`NYHXA`D`!^|B_~AB zoc(GwW@VL?DIldk-m2({4J|oZ<6UGWLqMrxNQ-q|C7Z}=hXQY7YaL#1wKRB!7u>Nn zc(|7FF}Ob+Ew!&Rwfzy4b~-L|gok_Sr4-ZM%b3)^(&CN#l_7Pkw83e|QpZV%JNQ)> z0kDm6oaqAi*-D0W-d5@ru)=JsWfXib-<;uSljkme0FabvD47{}oSl%tN(3{aj?RW98hyMsGaDr~(tK7+v%0citJqs-im09PW3bK%ytcJec)sRT;1Qm1!(8C*oZ4IC`mh&YK8{qjdl=QMoSB@I zFi0z<=<1eQr~Z`|FI>+IzG9{o4wY^qA+5I67_ ze-Z%S){#zt?5~KOrT1_ZU#TI22%jBzoSjh1PQx$|y!$Iw zJ;9-s^K4QO0&ziH`GU2*PF5s#v0hrxFK$<(d@h=5kb)zS9x1%qnozzeQhjh ztSBEXc9khx({*jRR|_Tz0!?o^S9QLn+84d)E893**L{ViDE7KV>k3_%M&T-BOz^x0 zj_?RiIAe%#JFpyu()|)meBaqP-$79}W#86a-@sPywN@FOSu*Qi-)a!|MWfQ-Z|^< z{LXz8_wH3wQ4v#cN0Z@tm0pZcE{of zNZ_cu9H)Tl3HE+nb)_a_voEL=?LRc-1xI7N;CLXx(cR|99ea0Owx!kr;LG#-Qa%|} zbJ)!M)G^be0Pc6t5!XbK!wD|dS8GNNJa-cJ&s5n9{t?r|A;?}xEcj+<92U?&YZL9# zezA(=zSAQ08WU{zx8p&us(q@eVn>gigjwlZoH%A?3OlNA3bb|HO+!q-?!0fG#1C9b z(jWY_FPZw(W2Prg>KGdXrRK)*pMU;0_RvDxwIw-|{gd@48Zm);BHhp!B+wFnDO6TQ z2gXr}#3ue6%3!-{LSE@@+$SdX7a-52N9>y zKe;sI>VrlgL6d7Fa<*q=Goq6lEu03}U%a^B^?J2I0mD5p%$U7f-1Ee@6tj@R*ab*? z6ley6h7-YR_Q?f@#4`Q)GOVVbs%_so;S$(#cZFbZ2#{c>$(lm!v+t8DRWIn6*GLw1 zM+(sh0f9pMAYmjp0fcXRF<;(PoV~I^*++Sd?F1r9cYiel3_#;S*tpwV?~An;lXT@y z`#dR?!>!~kpH@Q<5uXj+@+xrde6c5Awr{dbCMfokI|D)iERmHbT3Fqg<5<27X2kBF z+0Vx&Mnm8T2CK1r@w8!d|+Ms}Qzvs7dXUW@N$x-I*xOZ9ghurl&G=sspfVI2`W{cU@zi8Z>j{E46 zQ9)AL7bi?FP{4>o>di}tQhB^VQ)<7~==w+g_94qt1Oy0A7!d_m1gw>wRbCp%bojQ; zJM8k^YE~Yrft|E!2yiR{8fL6NwCryTd@i&-T@ad{uz0hz|8VeXg95&{Z^<_H5i1)w zqv5Dg@O|R9tzkaRkkJ?t8ViHFx`LIx!Jb+_=QD9+^G^Abh1UJKoPDbPj1Y^(fzuc2 z-1)=bn>{VRUUR>8j3bJjd3NxM_zgyl$7B3KTC-1+<$;(&K<|HWq`>y>g`_cGFz zLBp#mlJcSI?82w$M?pXaPk{@w`3nZT+!rty3W>oZp%G#yj=280l0)Kl;rm^gla7li zr9JwLWiW6rsCqK5?*VOI%R!Hv52aWuextZg7hMLiWm+a>}54!Jj zHdkh}WUMO&syEq2KzLo zC2n-9XiKIfqedaYkwBdSm);!)5@qI$3OL`-?0vJ85w!1Wm;bc4C!2wre#IR8-> zVH+9CCJu{COKqytmaZ;l_FJnIj!c4~v93rzu*~gS-?@LmEM8kWU6}YZUG1)W@?zKj zDFfwT5qlk zCB-ckYi~b;O1*DM@`=`J+SR?5r?#X2+fG82N!_6yZLYAdOm|fk3-#F6;GkS;cxPG1 zk-WF`ADR`WOc9%BI{ z34wyaNH`n@ETH*fCgUQW`Da^pz>a*o=GU-*6vjnkxd0xH@C3a}g6h;!iOiA%lD}(3 zl3aG>?rzgv%R>vDL3L2(famGTq(t6J@^h?Fup9wK>P3QiAqnnCm=_Tm2`LuZKw+`T z4YJ$V2eK!sZTB}dImRM50t^WUUkIFoV2N9IwD{2n;&OiFa+L_1b0~Pq^_wu3AX47E z`qC!l(=94XMVxc?AtzZCI}8my7^FM2iw7UN@jA_vJ#%zlRH0Fz_u`hs?eUC`fP{8u zpDowh-aTwpcyl)ezyCtKo7ugi5)2YrJ3;w*^tRuvT~olLY+Fwrg(H{OW&UPdw$NB` zqPapFTi5xzZzZy9C3iV5ND2g--&wHzBFKO-IH}ij%LUqR4??XT$z!J;*?R$@FU&AKunDtcaOE=jiHO!2tJYTC2NS{2@4SVcycuUU* zA=@FXcKEA5v%!qiliKhbL*p0HV|S7IZbr-JE4>!aek1L~XnmmdQj9suz5C^#c;h@a z7mjvBL4JlSgTsP5H~7ksUY{9RQ^B-rgQKhOOV$?ij~i~@>toQ+azM8p2WV{Cq4yKz zk}S-kXS;7Y@#r1f?6pdOlNRC|g`6LM!CQ%L;L%9t+IyO)kb19@krT*RB&84%ikeEV zkFr+@O7gU7asyg-Ht~viM!j0A1W3=H%B>k1Ns&xfF=&kEQV?W;FQhP1G7(y#4(5G7 z()2MRO0M6{BNmX)()`Q{-okNNm<-YWv6HF>uNOXmP)5fLd*uYqR{NKOJGwX*w^?W&DGo z|3I5F7wBYgDroCVCDo_K<7uz`6CD_vnSdlBp=n7RV3z1SZ`!A4yTl=W;^%}&q}`8y zs1FQ-CPLaMy+VNlH}=RXx3qPs=J)ex%!1Gw8V@H>=CFQg4pLtvJgd;P|K^ygP*(TV z#NmHAlCtLsY|FlTHdN*$FUzT!+7nEcQiXfl8Lb!StP>nc7J|4+idmjT3Qw|(e{-^k zVsJzx)|J+G`}%kHg}Tml+uCLF3Em5w_Bw3i$;eUQuDjfOCWZ&kwTj*N@+~4bhZFrF zQYrZ`%fH(9Z`7;GeFv-l{+v+AdM>kjmc~pTH{%)S7^T4SHdlo-uPxtkXOYD3D2Vhp zj=25xXCR{{5zu(>U`*+A(f48wC)hu5r(d4R(y0ry&#PBqoR36*FBcpJji5fE?t-o3 zV~xp=pBhf6rK)$D`sFogF_a*|pfe#0k=z-B~urFM;Gw=?6iN%ce| z0UTy9>dqyn)m4w)5PZSY<9Wvom{}Nh$aB( zRh2{S`a^?P+j=Eq~E_n$eeA}B!U z{%BBei*H=Z)MD1UWb3Uzl|vU-h!^-MB<+M?hj=J5>V4&m1W#bK#cY7~)}IODf29{3 zT+b*gFv*Vu%&}yQ7Jd*t@@{j^0WHrhaV-lua&(pwY%O+v=-+j^s`R0#q;2?hp$wEg zchPEt0w(9}Eh92>adOxDl`Fe8?+-Ym6Sd_>QR3Ux!^zJ^AZ@iL9u-v2iBBb~wSk zB;n|TecH5F$3lr*fW@bzGvnBjl5WjXZ@d6mB_)0K>ECLoTh1{NZ=He>nfykwaq$Gw zgY)2>D$EMPL*wQ4)a4G=eBoUxGx{MLH@$Mr*6yQac@RXh7cKTQF|W7(93`3l>4N57 z^YwCJX1zs+|6wRV#;{BF>M?|6fqbt*<8al9_rfx?NFPD+$3r!p9xx8}?J~4N&Txe+ z6$>|fE6UnHC9ZHVZh(>l!nPmX3_iLwnS8@&Bqm-(i`ne9STU5qV)(n^Z3E*IqW5FG zB=3{a!^)-dEXR1n zY58wG3Hi;b_WsKB4uggcPMoA4_N7U^1)4Fr`I4z-p|9fqWQs_cgUi(oyKhRw|UB`%5CFcelpDk+LA9#`AUB^a2IA35K8UZ?Zkh^rkya zdV9s?i~HV>(+RL0z#|D5KSFDI+s42`pSQoSY>HtPDn9qQJ|O&-JENuj*a7_S<5D)2 z(Mc&-vhlog@;W)5ct(pOxKdJb62s~i?XQ`9xU2>TcX!?{4|Sg#VzhWP!V~TX_1n82 zp$Wi6$+_bis0~(M#8VsPP4gHv0f}8s;UOMezcf2-a)+(7JeT7`kcQ}!-m z{)M3gt|4ysqLAQ=eC46fJlR(i^oH-QC#PAe1u0q!ONr(i6<}Lmv+ke+C_stBV~={# z?wePcboehhbQQb_IZNA%k*;VGWyv%-lEUWMVAHdI+$%AsmUoki&nemw4GB(LXu_?3 zZ^r+sIF1PsFgnMBOd34uk6FJ$e9#0O7P=Dwp~FC^pSH#XhvihvFU%uDp6T@U+%)Ki zL^vV5p!0@yjH%`4>Ig+WV_%r+`vc~7zxqZP6}Y<<9(f|r7vee7ro8b++JnHx+~2ev zkdl`{dr|W;r~ic6W`FcIER>%&pAfyN^BX>IMuUITD=6SnN2L^eFlDQW^elIMVunY? z8+Y1mjT_pJj9=Ni07Oq#6**T;Y2JQeqh?%^4r>2N8(NLa36BWrDE?E;9*5+-!{Bw^ z2CCvy;S2Nv1pw!*o{*pP*W5}w%@MXcOJsjNaXWn#VBkhf+2WaIYl4ijKS7 z%xyvV9lNyW3OC#Fn=Vr{jwV6rZ5TMgNMJbx35tF2NCF1XP?)=_6|`sk!1DsE9>0>E zcdw5ReIYNX=3PsJ%)O@9nfOL`pJ6IJ>tI-u!%mN&U~0K3DRn0P8tZq&4+%uHjCxzP_HG9Xu7o^XS%$l`*vmAh^6;!N?>u25gP~;a@kb z{DykLEE^$TAHy0W9T$A z385t3N`fBzz1Bltq`U%=ehry^0)m){H(I`2Spt(F)#0(5O*h|0Um;*diFppJWWUE z>u_-O&%GvDDs(l1!BsF4gAq@x%dID6c1U=#^q zjzduaVHQUtgPU7r7KSA`A9!Kq=rA=qgCOC+b3NpDP1f_roiuWkCReWfqK{sD?M{tqdQQerYUOxgH`60z z+aS4jPpW+{)=<%89sLyZrmQQ=V-?g%o*6~qro7GDx{Fm!qV*1AMn3+c(*z`xYg;|r z!BP_4EurFN{&wr{X@5l-z2>fwkUI8HDF|XKEE`#BFDtM)$%JUI1aI0H%agYXV$fL6 zjWKA&~qd#p2D zR)6${yZc>^CYi1Dj_l%vb0sei(6Se5PbSS17ISO&aQj>lH>=scan;-UV3%LB_nTCV zvi<1A;WMOE@jnS!)7M8Y(K&GP!N3`f^Ih(STYi~!vh2;T7$zQUejowJLWg;y&p zgC`993kb>&YKREH$1~w}WEFgh;(KdyA!3DquLU64n8PMN6whd+x7U$_1~6}il4y4{ z?gTg?2*Pr)ha2XnAFr40*$_#5Gn@E{UZ8;K@AWR_>Jz-Prt@J2<67{tSQ0HtMuA;N zO;mtY#aVW-ZOnbful7UXkeaTPUn^#4kPd=kdrR$y^0-rR&x{^Bc43cPX!I$prWYsx z{bm2rqOcu*9`IJ49Qhnl@O$x&HBBtv9mpKsH@Kbi^7calT<;3q%1w$eOq%op1z42` zXyfKBd7RQ~q@GS8g{P0x zW95T$U)b14e>}a^RQ){H5lt^pfU)WIm(z8_-rPU>BEpols^3(V(}G>J>vF($eZ0o; z)>Mts(V2$}njxZ76^CPKxh%9798a05AX7iOAh2V4cv7^JBi>s5-0hAB*J;0w67|#4 zD=GkK-v3YxS>?H4`1#wc+os!vRj3Yw^g&{YC=wD4otEZz3nVP;`H&-iedN*k=)19& zUqa{CfGB{8c>nv9d~{_jyS8{nXQNuC!^S&t4**azOGX-VY|jD z4v$((qJUP1GL`}B**7v7=iz!|8MA8PCA9A!FqoCA0+^>4ChvU-OxftC&0@ZuXd{~f5;va8*k^z`5E;*WMjLv z`-*a>L*TMRpajEtYud;CQWw!a8kN2-RXP2RhI#w03-R#+uaHB+!Oryky51=vL(+SzFR?1 z;0!$xYTC`MJ;B&^jghevo|Q_9^1Ci(KhL2-`r95wvAa%w*qoL(4uWSMR9N$0F};Rd z+SWfsg3jia-Z6?7FuvmR&@Ca=!kcz7^735H z{bd$Na|(_4&8p2vYWCmlbrf^!rCr}r?sKT?5TMu=6P1_=gEw%M|ItWO?lWz_LS-wf z*<#GS@tfWS(MfOasK#{&%^Q%Xs#Y5m5YfeXG`j}ZoUM$nF|{`JPu%;tnQACGTuLeo ztV6gUCFKTda0@}gt~_X znAMhYWpw87?+40>bFfkktqo_lMm)OH(%1D5LjlJl8}sowgi;}mr2W-Hbz3enpRHZZ zd}tl)n|8!*VET3<|Gk62HD01SK3_L|O^O1d&82g$5+tW12E&Ha3WhVo`%@O_DJ8!O zR!I_>?>O*2qk^@pug$x8xAj}uAwfC^)^sOS*=W}dpZ0FP_?(GnzuvVCS(&W}cgnIl zDcBtO(pk4{%&RPXLU>9!)nJ>#3V~uj93J|z4{DTbLlbPDd!sB`^d!#hvX<)+LG7qD zDGHc~&uF}G*`&0@W!NhGg5FZ+nNph-3QZ(h`k_~W?}h2(ySsM1Z2j|Q!u5Wh!*KN~ zh=x{MuE{qWJTtsRO{|`&1b75C?o3pgVo=ZvT5plUAAj&;Tqf(CZxB0Ai2x?|;A|XP}$fwX@tEty~sQg(?_*}0&SDwheJVu2A;3CE*y6sJZhFu%~?8>o|@4eXR zNLAkC<%`4f(OZnGJ-zY>_bVEuBJ3we9G2&^E@yl{34151^ zh~4o5-T=cjAPOMWZghPS|EVLuwqfXC=;L2cwoB^#g;1+XhS^VL*X|27l1;2{Z)Zo! z_Ma4?7bw6t&@L#uJZ4%;?|vDx+Pn~F(W|Bv0Uf|#;HnONMw8N znJ@3vhBK=mbYa@8C3XH0e1lV6T71;5yvi20Q|ne66yWeocSCU6+55#En9HeSHZuHd z{O?x`3QvpjMu@f~MAKo-0r-H-02+%cnV^F^5^VnT9EnowiX3kI{0(<*!GjZZ}H`DG<-TA z&~j#gd4)XH%X>UANkSpKmXq~u z#C(lsh239}P;=&EJ8CB+< zIG3635MW~s&4Ho3%W}U*B#ZuX+xm_z_-)5W-zoL`i5ecEj0y#`!SWqMxGFZz*vUw{ zFdgK1=DO?NigE`R7fOK6BUF@X-Msg*@%zVfgXZE26o z7ZA%1lq#l01Vj?Sg$yP8vjsHkq$5k?2`6VaihgPz!XH}3)!OPnb324ys{?8|UdS8FA zp=eb8*z}N7}9rq3s8x20w0Td9x`G}3{NO9D$Hg1_u zZ0m$BvW~c{04P1L;*NJr-o&3Kv5qL@>_4>MM&YnN_3=nAa7BW?)fd_&^TM<+?Eo_H zh*jXlufB~Hf*Q0+pqvW= z?^MUs@;o=m%_s$9KG{gnJL+}S-9grwB>wO#liilDuSVCtLaWP~tzy)e=Am|GRcddK z6X(6r+?eE7!%LkT~@2I!Rx`9f>7?V-*K| zSQ7c(=bcMy5f&EGJYQhC&L#Yp(G0y^eNf4)LWTQvbndP>c=cFqYhqJYgD)$jJ0ylD;c(mr9{e{WP{(r9i~ zb$=(LZl{E4_|^@_4ft!HeKM~{c07u0pivZMtEauA64=C@gs**XBc(reXl?KKhoJyn9`++nNl{muN?bK(2k#bDD(u@z$H0ng zi`XZuvR&11TK^TePwi4E@6qz77bsxQ&uwO*srBOP=PN3-zNa{d$??z<0%R;2oYhF^ zt_(@Zy<1px8%c@b5kbO6IWvi={-Ot>$Sam|_@QoI4{nqNwU6<+3+(6}Dsn?jCO#cc zUkzz9fUr40Q1Cb}ittp`n^?hxZtC<=TT7yVrj$uDZG4^ZM;UbmtGTxy_CM&LrJ|_c z*-(=@>Rk^M!q_J3JwilAj}?<0+Ea&U9|@3&@a4}cbF(Z$%4D<3J(%|X;mzjr6*L!I zAxHwXv#c$W1~iYILt8vfbk1+#_H>rw3}jR&pi)2}ZmQf>YoyKkLA`VtcWk2}il#_z z>R>&cc#oX z65_Sz67;#B>(!U?SsUJzVb-qe4WojU6dX?nt3ta+UZBb}p?c^QA`*-0CENE#;UjiJzJTq!b`GIoQK09Fl+j`DN`?~h9_2X6$c(`@K z5u9AnP&>BRop9dy+1}RO%pYBTI=*M-MgnIYXyNd@!_a7O1^Z2XX7Wvc%2T~kW zY5z{iev`-$Qy-N);i>YKeX{CM9ztY;i)ln`>pu(yqzxYTEv=Orw$rsfyDgWad-$LZ z?ULP(`VBq_NpM?U44B5v3O5IhtNSIU)pY9paeK@5RP6s?6kusBuaVC6T^V8dTQFZT z4p$!DI=H;CQ7pj2Y5n|%wcM$(O?TIE6lHxF5*tGQ56w1ALgxg^S9qKOM*z70_;2)Y zu+k>8_9l`E6ik<(SL6{{`F4Exa8A{(7#iPAjkBCElmZuNa{%U@;+wM3)Wruj-I z%#bO0d@gU=e>_d|8XiZ4Bg@_imlU99fqT-ibdXUFD4?TL$ULFaWU_-hAbhyFftL?S zQId<7#0CIGSYCl(o;ew+GJN~;>9YlX7tET060AJ$?NfzA@fg5a!eB+2I4$i=%Da+n zNwDov{YYtmS@;)vQ{jp=s?N1R#-Vp5YSs2A>gwC(lpz!}AxkNV5-GVze@t>_s^HLJ z73~Ev?CKyk11ZcU#tj&TOIFsgB%v79dL3fzXM(mCmKddg zCkT|ili<3D*G_@@9@_5BabuqW66>M>M~o3TUCKRAb3{dmluTBaFq%5|wU_<$6meVk zTH%bk)`-}gV03xm<9?~mPoR;HZNvjcI8_MKv7^_Tai#Mk`n^Tlbq8%b3-)xndr^&W z98qxdJdqW9#01w>plrGh@_FhwlCK2&h0u0e=~&KyF|XNM=!!mB^_@+e_okp%oRVhe z_bqp;0opdJkk`%D_H?l1u=Y1$qyz5yZ18JtwGyYEh9XX@zsPO|>eDRNiaI61nKLb1 zHn58s@@OqC$74$7z!@c;*YPX@mo{54&!abw*=mjF(oJUXxSZVH)=lq%SMB!0as(zy zcHIK|GGWq{ugt;O1qUa~`NtA!!C%vLUjC)^3V582oC`QqUHktjL{U1SNK&FQm>DyO za;W4Gi4sYSnZYn-GzW*Iq=*tJ=Y$A_L}$^7l!{Uk)ss?0PB|Bn|K58JdwHJscU}MI zect!2>-v7~wf9>0y4St#wf10jbcnjTDq3DN4vE8I(LA^uDqBP`x!T+%G-I<^tDN_Z zZF}tw=wPg{sFLQzV6vzpp)aoC>WIx6l?3BdhOxjr|tW%{E&^nn$0u_^E$AfKSm1k2WkYygAi02`Tgl1Qn8( z7mMme;!rUxDw|8^u(jCUB8_)%yM|uOD0leg{qC@DFZqEvLwC? zdfh;(hd_AD-a*}()p~*zaYQUPBS-Um)+G^s0Z6OXhQWF|?zS#FjO^rxGPk&2 zm&EG>D##>;9l(H)jA9G~A0i`#G<4=KgUeuB>YG>m4msQaw z3Q-DW_HX{w>*_%NpfZtw_-}o3-z_$(Es&bGl9Rk}m9^(zen-M0e-el6jqzhrsIWn9 zPM^jG%i&K|cf}{lQqOm|uI2Uv4YQ~u3Wmd?QZarQd>|8)doZfE-;o6tw6SbQPV!-q=dXpu$!#uf}@?!R#= z?ZqWR!h3~37Tl4?{D4BJ-&VauFpeFvw#K7 zfEJHICj%^33S6RfU7>!K_vz@aVeRc_)3r0&0J$!P%LrsaGsuHyeWh8^?x@SjzRpUw zBhIR4N@12&jskK5h7HXi5*;xu^?TRVjp;4#XVz#8jhSavnr!&zGoU46D3C+L!i&x0 zaTjW?Kl178wagi=UbBBpO#u~RbucUvIT(INSlqTJ!=w#ozj@~mE3w6?*F>|<98W{V zI2{a`MfZe-wyc|2lY4}Z$Mb5~y_Y-oMKhE>0YWT>%wz;mS!@~;@z=Pj^MNm)$ek)a zJE!kggmY-oSF0RAjzjE1uD~GqArC#!`ZkJ|z1OF-{-%HTwz(QttH;FwJ)Wm0`N7(P zUe?YThLJhamgjFjE}=cEKPejlXm$Bo7TKE?fEa&TkK4rg@p%U}_e9#4=bsO?{WG%z z1@Jl;HZ+F*Y=06Nk=K^wydd2-ycs`}K)JSp<+H6J=@cs0)q!RRi%rEa{W;LfB4blt zv*KrROTbSDtw*n$#6B$UOeTn+1VIM_yAwN@&7mSM5Urv4>cCR=+narod(DWYDmK+K zY5+L_<3SB(GAI~!upgbq@I{)CW{!AWczf9164QLHKINKAF+1c6$`Ij);&@}Y92(DC z99}Iy^|`c`pDT{upEr3fz_vI75Ox}b<3s9-k6RB*FR074o4Po%eQDD!7sFlv!cBv?LTsPO z%W9p^vo76!7}cPtQG_ul{6`VM@BjwAmnSUc&pr)6 zy8mh`5SbPO=d9LRjYm%|Fne88TlypQ`=uB^03rYov@}tXId?2|zW4gz)FZi6Q=h26 zr_Wqx1vvl=5kn?%FoFQoe0y-J;-Oi{(jRZ*x;Gsk3F}-&0>pY4ZxY)Z!w;A6LXpz8 z<$)GQW2@BXop0(e#v&4>>407zcpxS;U+KIK8~ZXN;-jaU*fo&}wfScemt(fb)ooRHmR#Dn){XE*ru&LehzCr+YNix&V z4_YC4?}ARo$EMpy0{+arzH*zxy2-kEnz7H*rGy*~;y7HW2^!Duy}s++J|cR$_4aE_ zp?{re^M;Z(GocudxUxSJj%}VaFIa!{_8Jq}I_JhsC*4&dV`%B3vbz!iJr=|ECb6g# z1cUW0W^a_vsBAIabN{o^5jkRbJ=gXopvUp-1A+6Dj>{&NTg>R(L!C1|FIoAwEVip1 z0PvVV64^Hxcp^zr`6m_Sq*)OLL+P({U(f@SZL|TaQ!K~%m{JVop(2kxq0?H zbiY#kaZ!tq zx51)%$R+4ZHZ&8MP^g0C|74hG?%HjWqAa(}R=>ks{_*@_bgt>bQI<{ig@xIxal=Qy zN<^9!Naa>&OnzCWnouSb5|BX+;01A5ZP7O?@H}PF!Hsutdkl>egMas!HUnxbCXmSz zdhux9Z-Wb`+NuKgU8W9D>fg*;cmEt9#-XNp{|osgjJ1o2hj}n zk(ZsHmEdaJ27db+FE8~_MFD|8YYd9Ws_p5LDrQVsIWe=4xU6^p-*-_Q^sI5?D( zy{TkhCYJ*RCFqYBG6Z@fj=k}t>u%*sIP%ECja+%L!a2ENH2{eHKf#m&cIV2oySlSd z)HZ&8?fAA!&H5_<;{FYCVaT7T6vj}zs<`@y)KR@{4&9br+86-B{~Pl2h57^rXI%W_ zO3XSXYhI?mEjW`FVDdrQ765hs4MHolKg7-+6cEwa-HkkKhO06 zNFRWZI0-WwrA}N|Ub4Bcv;D~4c`H)4dd%@u02l*+fp$AIh9?q|hAJ1U96zMKSmOS| zo6qkycAREP$D-@F@BrxIvuRMNz_UahVW5-wD?Qsm~Fg$fD zGRzMk@Y6a2+Np4PYvp84|J6e!f3Akgkfvm_wJ^N}roI5Ci(v=SpuG(;w)4yZ#Qp5i(*aJkALAFpZ zVxy#$A(g6dbIMjL!Kq`$PUf~GaTI|<7QDrSMiZKubZCTl>FVu;%VH!y49>%yY8rca zV7@}W?hf=duxLc*d8`MC0;hg3h#8t}IXOzrzKQ)r4Drv;_v+hF0f1Nm$a7g1mG@Ve zq#QXmnL=1_fA|>XnPWW@_NtdFz71;+793Fr+r&>N zc?AKCAC1k10vF8VS0{8dgdH7!eR88iTg#w#qiw=UArG(CBnkzCK4`oOSHdvWyx~OV zuhlDFgjEbabVvo{SU`^91p8CT-Xw+>LTbhNud)}rXY(}HYf!z@TfepLiVHyE5EA+? zi|WVZjf2mT9TMBZ9q-M&Eau*=_%2yYW3V3}@CZR;aG)d_mOJIFNmAasIzw^hE~k&t z(iu8>JGKhta4JEDGAJhWuRom{Kt-|-L(wZ*Hm&w=4DW|~HI(K)pCs*z5+Hau`qJ6F ztpkS1@uu>EeB>JCqp4=!kF6Vcb7LOn@WExLtTF)x`)>wA4ILj+~9v zKC|tw$F-`N9~(>S02YU`6ewB?-0HYkiLIik9pS(M(PvAm&y@~pmdBzL7I;cu8l4U$ zKhTV%dD5uJ{4WaQ0GP^RF`B7P3aAHH|Lg^E4%+48S->W; z-!r0b-l?Gk7TzzDec^-@v2Uugr0D9iNb@}9z$9_?ezQwsK2L;dcVS=t|zezqEr zV);@yFXQkxCB=E(&d197TX78*W>rP0OQS1fZmq&NZq@B)DCOGC07%4Wuf>sNcdfB3gBs~!f)T-zJ*8qn(s^l)%vk;sCD zX2+Z zyZze%h={%nn@!@qPQ;&sR=;1~yE*%CcEUAIxmIvgY6yVn2@n#rG9!b%Cbc74Z)oY3 zA78u9xY3G426o#S1AxA;4KAHS!$4~n(oCynV^F8(`sV!ydRnk^N^a$fRAvK|fpBEt zojWUl-`~jiuL+gg8Bn%Sq4ebNx|-!%Ow*m@ARB!;FoHirUQDRmv}RoNJ=Vol4}<-_>(3vG&~9IrEgU4?YIZ% zb^r5(&_qWv%)Z(5+R00}C#v`cr{S|3(;Eskg#|P`fw!q(@izG%K31&FJm-J$oAI7Y zYInx|F6w)<3XsCR0Bwjkd;;T7U)Oh>zy++&nz*ubmFtzb*{+eK)c{7AhC!PS4$>X%%65A)%VzY1ZtES#iEGcy zvL$H=DIy$KkOE>rY5A5dJ_?r}RpA_I4oS7^s5zH~GCf|>giNIFc@L8E_Plp8!(?e* zOm_Qz{6iw3!wNGPV3K=GQd>`L+B#p}@1v6HPw|lB8L4XlhLAP4pRRiA$cCNME&mWzUvxA7Ck{zZr*P1t(ewAr2(X*Vf7jO zk`Am_qi;(V-BOV0+#cIn@?o3Dkz4QEc?TJYTx3*k=FY4)ubY$1<&OBj{6AM z!(#OH2?iLf7AJ@!qF=ex=kVpcO*7|tPx`%HeJ-6MAvsNpv@J~9vAKbBbj|4*jcYSS z?pB8EpJd2%Pm^;wp8AMkH+%8A8S8Q`Gw!r^v)8Pve|xF$IH1H~_H&sWUKn)p7GI?n zrq?Yk|Lg0S^G>}=ayL1E6c2CFcpKzhi+)+_6mb5o8(j2bURv|;2aSL>K&i`9qK`Xt z$nZ&z-S@DodTW(rsiy0nOBmJ)^l*=1kOM3@Ad`@u_4{NV@2bALe2Iee#kc-EKz*4$ggWQ?KFlq`|`5HRq|rfLu>7Qk(!@66gL&Wl>|X>%I)RlPBHP zPo4gGwifVCEZ-rZypk-KXfCop0&*s$cWoaWUnF8n(_NqOL>D0Vz7WVpHcH4r!iB%s z#Y$_3B6fB?kDK}R(btY+aVUV}y8<-`ith~GL*=_`j;(yZX9q>k;pOIW>MIkUyXnG? z$LWZqu0O7L_$ai6dZVFy&};>5yGm>aU=cg5<1q}bp9i$uA@R-s91i>Z)cHItE5z!` zgtocZdA%P1i4$~>(DTK7cSYpqwVm`4kj1avk2N2k4BMU@@O9@XTRQFdKdPnKxtdvm- zns8j^4{&^sK(Y^Bzr!0vs9l(C`}Q^NY1E|LQ(Q_`Rm8yc)Pw|OfW-@g9F5B0y_a&S zW%TDcQyp*T&vy^?`rvc=(ZmQq@O>hX$)X_Lw?*>jpMts#6>(QejY7V>!wx?)9{}V8 z!Q`j$j_f+NcABYUM$WoL8VA+>F~lyXsGbDm#OY%NpV~t&1rHev-`zA2VQH>YJ+JbD z_vce}TOM9BQv_f^lUw*C$R@Dgpr*&yO9DUqijkn|_+V zzAUYLtSa}%^&78^#7)Uq zS>I?5H}L)XYpabbU=GVS$6zv$fb?zi1`(SX^vKB|qdVWf)f=mNc?F^7up%z)d5Hm6 zv4Igw)^uM=KmVqIzHT{S4#zi#W-E)6lk82a9DQ;w*rqTW^K>S3j|Bq&o)4gIdS%Uh}PbNjx1i+X?K{XujL3`D{$@kLJ;yiYnwR!}?sJHp50@Xw{FO{~yE$RyHTerx*1;P3G9KBA7XV+NFD~vO&?NZI*eUW z&~vJc8?}g*QG8q8nCht6ln4-lF{Lsn{Gzhw_zq{~+kJ7e+RY4|cB{b;8*^jz>td}17f}Ew7-WC`f#%V-cfC^VDvQ-DD*n7QR#iP**ZCBX3#K_7b&w&vIp>eflcEj;W9?cTpJ$_xurk0E!_Sh_pTQ4QxgCR z#-8oX;txuS>F$2ROoz89qM~KmX$k0D`d(WMf$Tg?PMDYQtfb#u>xQqg^%Y z{?zRwXRHGV0!j#GwBc4oieh$yy3Omt=!9btQ!#CyKLP|192Hpq@>YK4mzw7;t1q`& ztk~r^SfFcL2IvLD&W2MRPh=9U51-L>c7EwIQKu_1^Ibor*tt)31B77Q1x1QDpDv4L z^A*L0Y%wy${^_cVS-HOEsn_f)@-pTq$Q` zkXktqWe<*HJdRHtq|bXgQQ66Sr&|6@^d$fbhToIqOU00&6`IZArAY~TMwhcHXK$un z3X1)?p~)Fq1uy`I&V4^7KaDP5{@0NH*K)qzb<4AVwG4YxPoJ7X34#v)PFQ%7yS?tG zn$^j<%f4it^CuE+JR1gN&;$9Q<-&FGm_QBVd&UpfhMxC*?m5@=?42s17>87E1!bJP ziWw(%&)>bKA=X+o=8bcd%$(8RfE2#(0U71re=u=Lo*&P>@Nk@a`J>L0(!yEv$F_hN zhvDm#6E$;PmJgVVh&nW>3@lSToUJ|!(BUzDP}b&0LQ1u1p^BS5b@egv8a{Lh&1V%K z<}?@sT3vq1K||JZv0ryP?TiD&2TY82)jzOFe{CMW0TmN)a4!HiWkzQ{D#g>SM55vY z_59B5m~}9{vPclAA+<>F$dmTvBVW|N)=Rg>So?^&&*~Od3J7pzfkR>mk0{)ZmB!`v z52T)UdHP4pk#$>ykS_^{v4RVE$c;(lD8v_guEAqBe)+7%;@0fBP3? z^~Ha_=GJx^KYIqtVs<5f;CM|4uSRWh?+C?)OkNE!33hV6k+m}SPWnPXj_1kAOcsla zj0k*m1HG>xpWT6byHLuTrdYao2-)R62jXR6WD`OjZ9xv<}y=`HD-MeqY z!%X7*op~{Uk^nstasY_oSK>-Ykyghyk$pP2nZun^s}oEmpf4*2pA@3>D;(Pmtmde>7}w!l@Q!$-b8eA%JXp_2fE=P|-#d;#X5?#cR-Hi^#RwP&Lp9u z)0}xz@d~l%{nbB_YwaSvtgbk1eGfozp_4}T6`pd+>gBKe%i%ItEi}&x93JwYzunuZ_YyS5?$p z6uY1NZm~#b>s!(`UqFf6UK5nmDx+~Le3#Wn8&*|iUm?Ax{Tn?J|_YCM005HEuD8JtLUYh#h%7pd!C)Ned}wVE8O59kSeJFW2aI!8ySX1D$kN#~&2crf_U_6XJOcEZBLVk+k_b`R-nZWEi(YX@c>{45Lw<-ziP@?xr5uoKDbL}=a}Bo zCKSNyfb$3=tlQ-!-3eAaSg^Tx-D^LZe}4Q>_9=LohQE7&DJ+IUrTU}ABH@rwo8NLO zo)ql4ap8&cSB)YqQj@Y6+M2Eo@_mO;J)HfDF29uQ7`B~M#N0jWlM{DwJE|t|)k625 zOT(Gl&+u*O#-5tl`m1^DQROlLBn^b}f@p+8bd3@(zN^hvylw6tv5Um{|n z{b5tkvl~bMvH%Rr!(io#t&VKPt4lI`?8XXLhF!Yx;NgSe7CrD(}3c4G5MwFkmv~?3!0ly#>QVem1N`HX)@}%JAP$F-#2S>Bj(&J2N?b{ zK{JQU%Q2VSeAs9|e$t$J^IOtU9ObJM(M1E$3!V&c6!ma7H|+D3vR~rf?AjGwaQwK& zwKD*ri}BzxD7@XprD4+ylJurEFaPZFJeI#gdy#mn7$7C^jpMwK?}jq%7jG+4tZCSz z9v>EPpe&__w89@y6ZvZN7RJz6zPsZaxBSMoaKmNd-H&>w%xh5qif?cRg-#$C`MF>H z(e;G&mgZ-Q!qj_ka*{cU!Yks?K|?HVl-ef# z+|E(QGi^t!D8S%RhQst{Vwfy`ZB~;#_#tHWa>qH7rh2|kp-SrkuSYcud-nd~$fTT_l#*UV)=%FLFnT~Uz(KI!*1vV+{Ff?c#w6Bub$h&A za(Tfrmv%jX)<@}SCFRFg)3fq>M_0-`c^>;n(qgCc{y!Z6W&rjHi+^pc=CV&(s`Zr? zcg>T(@>}Y%EV9JaPzEcwWhKZ4MK{0ibb8bLKwbr}WIqs=(ZD!T0>}ZcKrS$OK*0}9 z3EP!#EVoYNzPY*Vl#QC(NZ|p==qmsW_y?Qj=TGO~gWfTx%qw%vBZ)1~W<2kjA+aMr zxj-AB01x3;TKI3r%Z~nNOMJN*y=Qa|k#x`9dtvP^x;4m{u}zrV9BC@LDnYaC9QMI*PQpsbtwhpy6B}n^mfMeiBac= zGmGvwMeK@Mktkw*=lRQKfFbak056|zGP3y+CtdSKrIC2zTA|ETi{7em03q@b!H!cg zE8BkMY>oB4*UUGhI2&&i;_>$Kza6z+~UE@-t0HDtY zxc>0PPQew`-$UP~G(|p}$L*eZ@kh7XdS@PSnCDbmY5F!_|7adp)V0fXA z@jeygzIVs8y+*@9e#2> z+vQf%-;#U)6nFr9UmEIfAPsX#nwX<^$AE1X%xoz8`)pghgyBU1Bm(XRZMfmH88Q^k z5Er_`StREao4Fyh<6#ydZBA_dtFSF73guM7y`7*GA=28Cq~7^;o{h%01!^-cCcIHd zZAS?L^2rdLUyeU{vRO$gmGU~OX0f){Y?m#a?9`n?EtbEA3O^SS1d##p@v>Nq<@y`Z z4O_fOqJI`j(N$3jKG&tPp$YB5yQOeQQpusOYMK5qo7jnS2^wc*LxhK+M7Wsc=_l9* z`>~M%fZ6EO5c_WOtQ8)?qKfl1>CRV=hY1PfvLlm)p^~6vo&O0?n0<{;!w#x=z@BT1 zhQpSa)YQry699T}Cg6u06w2QD{n+~FN86G)2F|~)rQa_w_IIz@)(PX+J|4|j89G^i?LAvgSDfYWHKGpnjyL|5`q4{-xzAUOF>Rbbp7%(ZsD1(Y~c+3D7=xsgoux+EU?L;K@} zYJKfziGUK%Q$mY2ulQ;9U`LGi_n)(B?l1kdh-IjKdiHK3K(31#9w7FG+|-fsd|37D zjhal2Jo#g0H=rc&4GZsUJR1IEL}lOcv}D^CWi*Vn%21_*`LiBeRwW}-t$R}UGf^Tg zO47t9rC~pN`5n6j!jHf8kSq1@Mg&&Qk71N6tlRt6f3Cv@pF~BheZ1vWp%N)J33pS% zn@aOEYw7(940gP~R~{vWjf`8<;$jOhSb$-3{rvcA0{s!);4ioDd|`rib+Mkedc*Cm z%>ac1DD)EW*piwCn>IzQs=tzz@}D$40$v?s0T3Q+Dwu&h9ao=AIsU%P0=wzDzU;Pl zR~H%_hzAe?fC!h+v=zUOHst>Okn1-{TT&P+=6GD|G{6u62K}T0gMa_C$5`8ap?C-J z+d^-TrFG)R07wsjpuhmf1*EQGA87Z|>PUs$$rBCDCGni$@loq4plI6OL}1mRbmur#)3-sK8bP>N!j+-@bO*p}G|4W@$03@JtAjJvbX z{*4}uKUv;$e&vqf$4^h5159B7Q(u2NsK(ZkJg)xpGD!Bk8Dp{a!2&>v1Eh|1k?u?W zsoCG&l6P#NIrz`uqdUg{DIUJ=D){tNP#}!Z7~OViQT(KldgjbI%oBYoc6Gl1MpvMR z0ny@#Gu}xGp((mH>tq<{}g>nRmYid7DG>-Ig-e%dIN8`7;R~ z1$i{=kwO3yq%dSIl(hSSSJzvT5AM^Rqb|L>yz6|y@2^@vKS#I&AVJt3n@Q*8WYT4? zm#^60vgDe(fvh}s-rSBezs~^#aavO_V1IGPr`_R&PS!cL?84$9%s#xD9smfkSorI1 z;bkk!;a2Cgy70_TXQh{)bE%sBbcy%|fDt6J{1-vGK%>zsxir*e(du_ks~CD^j_l>j zCQ(GNFXS+p$gZiwyQ&*warvtDGxarjm7=Rh57l@I)q1?|sQJ~+;R5lkc1xH1-91yR zQ^#)m*s+nzmjNl3FU4>f{QpB@j^HOVTnaz0{;qZw=Uq0xmT_eCqQ$RY!n@^q`uy9j zf~V^K=T(|=*;u)y!k;}Yo%ilcdao9qN$DeJQv4r+fyG;1xOZ901q75FXsy}$(ERY) zeH&Dto^YdZngZ&oF`?9*uHXD1wm)i%ur<~dZDDkLrVWrzL`jw9ylK5Ht*a)jdSe~~YZC;6gv~J9aIHa;R-d%ZISfDn5-yAu7pu$INu(=6$pG@(4 zG)i^Uz}<|UsvWyuuM}QnHh}&h_$eB6@!UOkrgIxRL;j=!G3ubhr>0n=*oOdt6%zg| zCa)5ICL#4lQU(4!<8Dcbaz#>ozCK--B7b`{d@CLG)&_VzA@4`?V)0%0v|_B(#1l_3 zN34?AjXN0Y+cQctP#JKohQE;f&ND6hiW`gv@6_;@MHL1^{4yHiCRPBfzQ@ z#>UKZ++-G(&GY&sl6NFD@O?CjVNrMC!XF(4_@Pda#n|Q&Yn@`()dh1K^JBWjIPrl+ zfFAWq_{Wye+W+mJ8EVnH>^eEA_FsEK4!2g<0_!dR>5Bh)Zob%G6 zt#3x|`j7XkJl6njiZuXQYJT86@$;-tlT|m!)I15d_DVItI_ixiF2|d8-HPQb{2{Sj)PiB7V2B>|(iTNcHf+8J{o`OtQ$8~0(bsYy2ZKOxOVYFcfbbm-{F`-rBgY`&#I6p6fnla#(H}wbCbhpof_L09DXk=7JP5^ zNG;mWM3HI@iDhrvGuRz7LqFxOTl}nkr^|hyG1O=IpI{0)54LSp7+bptle}c|XbEMr zTXu4-Y_1BxqK-@UqX6}autSzs>xa9elIOo@DnF(2$bA=E6uyIiq$;0y&i+S75-)t81JKGlJ@jCx<4Kz@V6n->1{Kf=7;Be}uQX+dxVdOVc07N~O zcViH!p{1?LbD_*v++y-k1MAlN_Gd$BS_eR)-sr(%`cfGnO5b+6gqZNA`$uGS2xs%= ziHCFc1CESGy^=v?!=niQRqz8jLsfIhw?~fEeT)w_6!q^HYxZ=I0a(--`PW^M4_jbR zYD%{5oJ5rU>iy53^=U>$UUCZF3V?tc!a0Dz19`74=q>xO_JKC>!OKLI8%a-WBRa50 z2G$Eb2Tu^W_+53vkhJ=hhKcW0!or{2qOd6`pba80-N=w@a{jM&*c;@&T-!IlXrPoJ_X0O)%&s|JAroe{xc)_c>?`7&%9qWs&#wzI_F4Fy~Oxz6=T(wtnfYI0EoIO zgULX?eET2nI;5AFUi&o0M9TJEv*IT^4fpmh>rflIIzT`13r4dD_I}nE2XCGEB2w{o zzVDA^)S0^g0y{m3_|vJ=uGSdUdnt79Qcw6|^;gop{*SW4p2PogaO20{<$GoOBkA; zKmQJ7xMIi4Q)hm6y{;%o@Nn;JS{!7)0MKK3dR{BAoiR4$ z2+x@ebkXNRz63@bGk}-JQpUD6xjgir_jF&(!Q<}_Js$e5wHW}h{{jVv2t^mA6_@vq z?WSsT^@(c4(1=xZew`UHvtoZiUK`js6KG@=to2P$xq!Lmy7K1IckQ%+_- z*k!Ho{KvA61*l!T$l%Z9V+-DWr!}?L=SO6o`x*E_OZZy>25>8P^mVdM0 zG}2xcK9E25{{S1u3cu#;0f4+uc${sPc_7r=`^RU9Y*{~L8Oy||>`RvHYsyZWrNkJ+ zOlHQIg=~!_x+Pgo`cjrGNknN#_N`Kg(PC+_i$u2&Nv_}cUUl!U-*5hyf6nW?pXWTE z=e*DJE$BKa#dx1Z^F6+hZlRZvk#`=6hwdeO$Qr$?rKzc>qk~i@qVWVGnhev|32A6H z3r=)^ACMW$Bu&^dtcp`<`cV8q1&1FI&H|U|&mA511;iaTjMhnQ5h0pDrmfTABNh%0uyO{S=i7A8Up1OfkR!NS^6RGMhE2o~9f3F;19y6l|wBXtXYGsMLqnNimBpXsJ_V!P8nQ9x?Y;H#TQpso< zP8|cY|HPj+8`NVvop{cf&%)ahDH6#CCg8Uq{*J6d&I{d{ZR+dlTjW#j8Y!TvK^}*q ztd1u8heT6|uqq~}@VSCM)#lUkOu}B{h~iN>y8C_dM+*XfKX#rsYe=6KHsn8k&N^;X|4F-K%?9JB*H)KEFQ?Q>W5)qn{zd&0~i}+0iL%^a* zY!R+@Oy?~6nMK2RC8|{s({fbUn+3|b#i2>b(|K;abNi_4TW<6Fyii%Ky0+wy`-HzV~sSB!@C1wzDUjwWwJQz zvuuh~zZ`G1zSuGap75GNYy0_INXLqFDjdG=T|VQQX0gj^07&r}IN@kCvYJ1Z_TO4A zyf>k6`a5<{}RxvyYPgC>rWy5yIU>Iq4SpUj^wMeplbAZbEsu&Q= zw;f6XE%}8ZkItv|hdpz}wV}_iFZ+Y5e1%qNnDFM>X*K>o#saD+H>BuFS+7t0Cckmr zsAK}3#tu;C9q3`Q=ndBsGAT*aXQ4iYndX2Perc#^RwKNccrSeQzTxeKx*~ojDsu@` z@ZW|Ohp6APlN<_Ph;%0OrHI*d-|fu>!2;$`Ra#rXy)O7Emp4S)%a?Lin*@_Jz#{<% zDCnjv^@_?{0mQDp1_!Xa@7(mo`@IejQFf9GMV(i}3u?tAGJsD|79zEe=l9{X4)|P{KT2 z@}k#-Fs121+46k$wL0rhkFmFZONW9b2GKV1fn8`CNz)+i@GY-jT{6107&TFj1ucSB zkayncYyp*&KXax{co6|g8XBM9>;e)(U5o>(xe+B-2b;;GMT4#THTm<+RVRN_r{aPr zG(u1STmRHP(N7Zj9f(%fC3@_~^6>z#t6%>w9s17|g*O&d#Z{TvYUY)S_t_(Yo!xn} z{=ETUgfpQVQbgH&Nc4-V!li>5_AOe%Kk5lUSi}i3!SYF46duaJ#3c8L)trp;Pi$%d z6cH+r5jBVMl0G}EIC%aC`K_x>CrIYe)-(6hK(MGZ#9+L(T&XVItufF1Zr7V(7xqPZ z1muZIdl{m-W84}eEqMpDI6?r{C<)sk_P?`N$H9=6QA(KaiY?+1EYF&6jeAmRmTP}O zBtODNnJ4a*WZ5Y*s&?*-jk7A*U3#FG5(cEi;viKGuC|d24^I?)-r+v3_n`!AWZMuYNXh4D*-dI>$(7Z=IaTiO}*o&h1ZL{UZ~CrEJJZ?D!B`U7;Tf zGi$$){h>1Q#Zfy2?Ozl*jEzh>ZofT8Pw%RL6z&d*%G7X4^)3Lt9R<*55xlBKF87!& zLH^i!GQ#$h)ch+jwx_s$_eJ%E0Ga#DMt$7H(0crU+6`)-<43enF+w|N>Ymr{Iy(TTUQq-xRxCfdY#Nu7+P5MZc=rsFXt?+6Uk&hs^O@~czB>mr~G zDaCL;LEH8fD{**|UW?O1AIek_+G4W#(E|Q!_R&%2JVyvqbxVi`HjgShJze8Z*8U5i zlCcM<&&Y`i+V|%_V^PrL4a=JF59Q$LH@PHtynU%+ z_n!On)sO4Ig`J0?qcM8Mj&Cyl&|G<19dg$#R%GjnGWfEy0rbfqg0`m-lZfKIox=>y zhi=pB8QSCp4AWS_0S<8mA9EaL>Y*ZIajQem5@-Xp&rF$FC^{Sx?oT!1PC=#56pt7v zmIW(j*2i)H#a(_zMZDu5d%5Q_yz{xe9^S!XCkw)g{yqjVC>&29pxG{q)`E>rDyRp& zi%GR}WDcG^HL5CDMZ5yCcj-e#e_ExcweXngG58WXg$H&0eq4?L)4TkjgLx_~y2`K2 z>Lq6{dc@w^LA{3>V{6Jgj%~5iH6qH>sY)f4Fth^#G&QxLx$+%qKFGT ziW43!)!ryY<9PPMb~7BTw17MK?X#Q|90qgKf7tY%m)#}gW-Zxfg~6%$VF@%eyP|*6 zE`A`Q0xKd(epQG+)DC+wLTSv;d!pfxue8owYeCVi`j0~7ORw$@b3xu!;H+{tDJ-}bZ$Map*wSuKi#2K3M>rdyq%n8>0)PT*P6s$j$?V>Gm zr-<%Jh<^R=J}5+1fM&7-)b)$ld?(w=^7~b;nbAbnCy<<5aZ`pc0|j8Ks)nFoDI4qO z^tQE{9w})$z--9#nT&RKiL%oG4XS#KZdBRTq_6I|39IL6FHSpNjbKRn{1OkrQ}IC? zZ%762+54O}3=}FqmL)cp+IKo(pX^VH&K^D4hL0xsvE!_pTvAJ1KqiK>=CdQ02~X_o zSnvNI5C%sxd__B`T*E%| zD>H<`9NnMFG>R|XpDjC6Om_4xn)ce8Zc-h^G8yl*QCkFBnqmNh@>?S<=Ux`o zc1$1^Mf7)1G^%((xv~Il@9LBG~tn_jYj;O7HC*1?N@)8{NcTDX6lF4 zP_x&-?tpAu_oQne^P$!Z+?2gIuf;q%ZylnqdhL9hr-g!)#DtsVPZ9Q%K_#KM`(^IrQ-x=$O_CwRK zYE&$R{c5r2TmK+-z=(Zo2@$nHiAg=V$jc!Bs-8FH_{9@~(3`c!z|P9n9ky#TFgrQT zo%Y3CH8!vE#?nuaiVoQ5qxrd{snlPb=r9#R9zbYaM7Y{Q4~rdMT{+bQDmu6i|iyMJEii?%pJO;`_;d{bLZUWob#OLoJ&DoUPVbsRMr7ahLg!e zv>kBEV=KJBC@ipzFF{^RI+}vjheU7{7HuMEJ>VWm$Py1zEBindQ%;AK@2V@74uO zoD!>S2`g$=74U-gigcd?ALN<1aB<6S;b;7PNA4R1h4jln`vY`eYr+%px>ZZ^nrn{L z?>`5;e59|vfc8p2f4(Poe;gm23I8>{QKY1c7praj2HGpr{rH$BsX>T44&IG&%Dizb zX-53$4QQ`I_a&nInKm7QW|>04vpL^dzh7Rvm;voo>E2<}2kC1L!Qp`$^De8lRAs!l z_(}@e)AXmA;+)nOdlVu!|KTR3DY)3{28Tmw1xEC58}Mx@9t*iA`Y_jg_HIt^v+1$@ z&>o0>o4>g{#zZsL)_5^LYVmSrM(&pb&>o0>Ac8&LlAB%I+(Rb%e)-&xOrg_NXb(ic zTMkw8yeTDKt?}0nd7O7^T91u7v~zMK z{!*QccQ1qk6^X5FOGXQ)pgldB-&BZ%IH$V?Vda@R7WdQpj~;per4<=|bc$OnRmXpx z-O^;=?DhYoYziD2pgj;x&!0-$Ol(o*6#cn1<+UnRg(cS%pgj=H%nq@87mSR|bmu44 zH(YWLe$KLI3EBhEq}0z)=FT8kJCy?7X>s5v=d$cXpgj=HVaXlD_@et?IscP+Afs)z zVC&)O0`2KqF=XSWqoK#~Q$vZYczaFP*`Vi8`T!$3KC}Aok3P$grNGyc1|r(EbW=LM zLVMu(hQ<9=d7apjtCk~!@u_jw)7=ckp*;|trY-(x)18`cc-w@-&rGs>Q#y5T53~oO zlVhl%muwfaaJrI^5p`klO7XCM6SN1S<1^XZ(8%)3UW|RFI^Q{H?6t+$Bxn!Jwe=6A zWqy2}x*-=!eFsi#AD9|2Q>2m1x0u_or!2t&Dk;e7o>OjavL;Xb+6mi$?i=thNYY{_V58Uw_PY zs%gb>LwjJfv<_*VeYC4}u31DVIiZ=NclQM;kEEZY`)qVz{jFwKw_KrZ ztk9mOClwFE$+*GRsV7ap)RaqWDPItfd1^WHcnt^=!d#c;=0wQRw>@p@s=p@Qf%ZW3 z)rCA+N2X!n+?Qkao%+3BwwrwyERbcC2pU%(5U$H9^_~#Gomm5Q-%M%7d$dKKD0PA*nXyVckVrC4@AFt*s6g0 zm6}!7;IrCW!6mifdIC1ko~ES~ckj6_JZGgfG~2R8LU`lMlMu+ftip)qF~n1Z-gMRW zGj&7k^KWb{aT^b`L3<#Y=UE?Hy1}{MhMk($@XF7uo~SY!*Z9RIt)W*S#>E#&@uIQ}M%G zQ)mxFGyZ0ggIXGIL)zAS+pVTLWhX=z6`?&4AH-PiQggM+QLaa8RTGLu1YCGg5EZ8i zqXO&lo!&kNdbt7%K5rN=H{?^QbZ(FNNF^<6^ zd>pWD-RQ0(&Nxm54`Qn>zx||4msC$Ra#P76P#Dfe*fy~gTyk3tuY z3#MXjt3FY>2a$P}^GN1;9^)XE&wU)I-u&+u)&$m8NclnO#^j>$l2G&xDJ#je;Y6m-7EhygLdL!xY5&2fBVBh1G zA5eO^!Wd8=xDb(Iu64llaF$~J^$o2NCJ<){m6ZzPjYtWv6+#Oil=(m=AOw-fzMuk8 z@2IR)7{*fn6_2w0_|{D-NU7d08@)a{4KkX`6$W>v-Y=qe=WJDH;>90glE#WJ@DL}z z$8_;D8fn1NNZ2EN?e5v>C+?izbt)XJ->Q+1`f$iaB7#Zn?2B2?fxx|@2cf6loVp~ww4aMfXWlLCw zSOg4%CXoTS_Q!pQ~zV=Gx&gSNv729k-QpMh4Ryy(kElLtmlNS3h+`^18&0}_WM!pTS&ffiPJ z6CR2Gyw%PPy_*bH4tlnWUD_5l*9mR{4%Z!*pQF54rAW5$GW_7TsprmUSn;yO+uPys zPB1qTM%JE0V(HxxwoaqfY-)Ra>cda=)X>xI zx?AHV=9nMT+dnsE>}%vG3d${$16u+C1H3oMjNIDo9l{;qRp)9lBb2`MV51Ef673z) zSOT614FCI-{;_o&m&eE4KRP~NN)T5#8D_m&y8u17qkF1*GrP#(&zdIx+j-uT)p}h3 zKS030;~;8>CL!&ChuVKhl6hvV~Pd^fR8*>3EdW8!tDL4cMcxKj@fyb5Vb`nEv zuuIqAS0&$VWUUrhwbC%4Airbwt`b!DzR8bZz2m zqyh6{QRO1>vilwX`HtJ~lyWEdKU zK)TUe*tH!~DU5HuydRgWWR73Zm*#%v0nq~3>kJi!?`$7F$Fi#qJ-xqr*yg=i46`K? z-C+c@y%Vt35nH(Sj#9X!O-Y@Vekk8iZF&8FIB*mi|D6>UW$mx-Jojp8Pqv>#QQR)D zM3eD&3~*J(nInxUQ65;V7uqZ^=j3s%)@U9qF-SNL7?&Wm+^z@b<zT7XOCq4@e3RWWjW5y&Tk(&MJ*zKHq z-)=E}r_NxDC#O`tO`DF?npdyF94>?lAuHebx(imf{aras2>GB*~tL zwgcjvtI$zo;4Yl^%c|?FY%s}5*`jI_*wXhE>cRp7Vl<)bHd+>T&>{5tI>d?Ll1#&> zo2&KTIe#;@=}y#@u_(##wGpb;vtL_f4NX_;l>wiso7~IcLzXa%s6#HKo!#pR50+Ym z>2A||UUBX{uZe28{fYdwZ{O8%i3YB6E3-HvrA)}sPt+&zaK%JSj@-GY>=(fUj)IfO zFkl4fJX>7y*QS3gq@2B`I-RASJoR3!>F|TVNvrD}PO+>>+cD~)g6||=iW-6; z8bd}B!AVqchvT=2f2VBrhA6 z^?0Y6l|E`eICqRkJFTKJ8{{19>8dj$I8=&Is&r;*y&~To3w6D3!)0KFq-Gg7aXI0j zft}rdN^4N!0KT-iDJV+#{>Y!&&aJuNMnr1uN6)opm96+LQ^C-J?@C2WQdjOR?jnL6 z1x;rSFCz<22j%VCmtD!0%pL3Z!Rxy+h&jSZj*JKZb9#nz@mdv*JAd%V#H&1S-PkG1 z-(Zf!0KeW|^wv0Z880SBBEwygz%#`@?M^nfvpT9H zaMvP;&%4y`Lqin~f0!7FHq$AL2-r%8Sf3i6 zN5m9{qQo`_uaP@;`~X;!KuL3*2dl_)_NZ?*z2lainDbwL?R!Cv=%_s=dzr2q7 zG5SEWLvT-cNRP7=zo@$hGhWG9 zw-7!pEb9paGxw{fv78%xkDFg4wi^G#sU^nWALzUhk(tBE;b6qqwpS5P@NHLU*)4|-hQLQq-_Y5NY0 zda48Fo!cneL{@;Y!_SPfGbXQjnqDahJcDWhq|9%ajpm5L36%1G-f7ngO#*Utu7I%< z84gU-n>D<;nsp?xj0-%e+#7QBZCv*9GsERqyV|elKy24V5Q0nDT={#4WV38|!P14U zV$5y#T~I{9Uwc(13&@(f7N$t^fE12)P$nV)d--u{Uhs&Zuti6>#V*#L##6|QPLQQ7 zc&5BAd;HFQZvODxSXmtW;Dm=h^DB)#`Y;PKC=FymYHifuo0u3bWGFP~7aaI-c}>E)$m%iWIJva`v?kVRDQ#Z_!k2eIjOGCf-+n z#wGM(zl(NY1;{y5sMQ6i#QDhPHOO~l^VdqQn)}1~Mz*{i-*$sI5eY~9T}#fkKlxSq zvt>KF^BdR3?bdrOZEV7XS5iiYuyfNf{Q>ixQLh^ADP1hG;8?H7B<+#Zle&%7=AIxf!t*t<_B2nSvyN23hcc&F3s> z0;ctv#EoFqpq<(#2yLzTGg^;NSq0}0Jl^$d;e72X&>1|w|B~p4}ES<6<2H+4q0XCYYqt7T1yEr z_n3MOHiPJync7_mS0de+F_JOIF7i^YY~pPnN>N~4uQBgSAtPF4!4kMsv0M}(OY&jd zx9yn!@^q2ad*{%}0Fa`UCNLyz8>8!%r*{g9THd#~vOVvMJQ4TJ=hw)=)p}*r)FXHM z$?UgkN!^~>sruO4=|V~F7BF_CdNKv5z_w2sUKoxxVm&e&y+z>w`*g{}H8M;V$rvjh zjW*u;BoXAJwhhUnE||WQOKu8Gkh;W{C#{aY#y#r~v!9x5H1sCM%VB|h^Q){YxZV#M zGeeAhG?+{E!^Zenz>0SU9NVy(|J_0RB`q_4D{z*gYjZT7>^T%3v!`o*wO%gtXLO=F z9qsFO@?J2Il#^MT)V@(uYL%tCWBGi2Stf4>@bF7V+7i9qg!^6)+2YB!n!G zxVV0v%J~zlElD>z*A&_^Lq=ZOfrytIbV(`B-2dxkg!+!2(q`><%#ch0vcCTukJ5(Y zF-IQvRAv77$ld(4P|b-+643+_b!>qSP}oZbU^2~{%EW$MdGr{iE$nFuJ`9j3c33=u z!YCcQS~Yr&o`p3ROYIOdoilK3j2s2;J|q|5cr6yM7AMlU-+kh_+e!JJ+*kJn-Vngd z9czcjpzT+pYJbP<`i6ts-m-fsUR8HDRlbt>b3e#0@9aNGid)lJ&v(74Kz*&Yd)V+L zKTd;*5&x#zPqnxWnJ_qb1r0V3id^>eZ#E;J-$%ge>r%MS7~IsV!l~O@4cA}s2-+?4-{~mP^I{~RYEt}E0vUTk)VIEHja2c!emQgz>htTev(xeQoJ~Sa zXqA2GI&;{6HE>A+TPb$gtlq8(6MNE!in%+vmST5`b!OuYrdAj{b>9MCg1%^lQv%ry zIZ?kuIpXXNWthB@Vf(kmO2#N0cvyErFsYSY>AZf}zEaYg_FW?&?g9sHZ2D$OZV6W# z)|kNuSeA~ove>=%7y>gS5sh;H`_u}!fMR3wnr80;7OtMLLkCr^a$NbBFHaZ$FPm7E zuCA-R!E%zFosu4U?axMVAx%B~z^F+B@5;!>`-10{_r+{U=ZozN6ImEitC`ygzUFmx zq*Xs~oCBjksp>Aibwm~QQ7^Kh9UIhX$!#PHzQT1SqRF5tx+3V|=^NesdG8G$#37&8yYH3H?H}YS1hzcHFbXDTh8S4gZG4^;OwTBcko`{A6Dft^{1kKT?d;QkG0^MvLG`0IO~n>|iYi&hnQi#fTMN3Ua>Wy!sCAz! z&~Is0kwg3@TOusgG5`H>^&bFP++z3FCOgYo zeo9Wu$rh|QgnG1Ag7M2}1vOwQ5kgX`6yCht2*-$xyFKmMOf z>uETmgwX+7L4<_^rHf?P+Ao};$rD8yh8rK8INtcD_n`X zb+EymQmdb>L9*-Y&Ch6E>s@8(#XQX>B7VYc!|RPC6WF<09Sm~Vuf+e1Djcw@jZo8}6**-6U|5Ie z2^RA@;e=(e%$VAvV6Hk;98(%oNZEVZ__Xk{+#<2M^-g;G$(ATkf5Xz>Y7oh zDiWZ4VK61}WwjL2VJA@w<-cR+g)xawanAR=o$umBn8_;Cdw|;{j&3)fd9wcaY4c&n zvuPJy$l7l zVbzo+>ma#};ZBmBYjT0(Pqwkoa7IVNa>a150H^)(lE41yGjJTs-8_o4CN3*MN8~?@ zgrY)`nY`ol=6)K&-i9oSv5{BlcjAYtJoQdEXRe88KwZdW$Awh&$?YUbr}@8gp?rTB zT%OHC3FL&H4tW@a4<1{X$!G%S`LqfN(1VYUY5>2%*7xpafq%z522YQnaALM*R9qR* zlgKx-OZUSrVT57m;al6yM%7i+RGMll*s=9ek^Qc0MEqp}*#OJwHM-B)Z0m6QpBr3u zwtfS^AGUbTwz2|voO8)fF3nBND@iQL%+J#gO3m@hFG(%dHPkcHGXcVqiV^_a=nKlT zwgz~d&0Fhp+qe<`Ed3R*ZsJff<=Af8Tu+u~oSbt_FNtpwr)isJIFtxktSM6A!?v!; z|K5E7AV5-%bJKQu9~=>n#bUAh+Xryp#pyh$A}I#5Do)Bc-Cqv+-{hsOth{KLmdYY6T$ zTPzCPUm{a80NR3*c2lin6wEfIEKDctBY69#)8N^~U&Xw-mBv z0EcGb>nx5$7)5ul!~goi-n=i`dnaBx_5!~_R`3_}_;wy9NjCQmeVURjyw-%_kVlVC z6oIKQ0;gu22#qZ6Uc3+}(1@f6JwcdDqWP3ga&H$b+c#TOBVk@|C08rJ zK~HxzI3paPf&2}pd;nix`n@Qlg$iUBz$K@?Tf{Q&z#;?F{l=EW*bqgwaV7}pfElNz z9caUitF6m0Pd4K0{P}<;VIv}Tnt21lPC{>yA$qKG|xLEzg6^KT_Vv3)6cz?21QO z?)Zy$(aZtTT*Z!siR7Qb;XqH8g6~-5*$ViUkjwBeZz#eAMmLbofEmXNioKTLbLt=* zKF+V&0S^Z4GN2?<2do{DDyhyrL}o{)0Rv5E%#9>f#KR0wM2MM{r)KIn)fc7#HO3d^ zW-ZfjMFYnt_%-B!atIN@f$@q9038lMX#rs(({}h-$DSMs11wz@;`7uwPb)AzEh&Tq zL2m?r7rIu8F!O8xek*!Vrmcx6!Eo8xPwIjN&^1JQ3dpIjzYCn zz^6edZ(joq1O_Ic+9545!SNyD1_iXWM0Xv5T7wVtV5E*d?)>SWgVWc4L)U0d8QbZ! z8@UGct>R)8mhwtYen(Jt_o;+4%0QKpFcAd ziirC?mtWA93V;k#YSd2{$HDGhs%J|I+$Zl43ncou=zM@PY9oIgXl<#pk24Cf<5ZpM zKwBV9!eR}RvGu^GIH5@b!bzobiP34?*H?_I3u+V))Z1%Bpq3%ww~IsZM1^WGr8FW2 z1hAk}#Cv-xgkwb{wEYl&+*i8Vb5y=Dz}H%+WI&Q{C;9bO9Dgp;YNbRIkS>tnkgKkZ zr_)K-wHwAQh!oR&TrRaO68IySAg?}T5I2ych%5XTTOocZ5j>Cr97$a63-Qh={P>I} z46lW9OOU8U<|yNIj0GN_a4His$vOceLWVv-y=|W4HwgoH*&48rF*E1&VK$ux5oq`a zVfruoNx$4}hlDA*>Q^TnMo{F&8gkP}@yt8sBgMcQA4ObP&BjTlpgxu-ABpYpyG zc>MC(9O!LeH)l2QV|=2D)LRh`rDgbzQ}Ssn9GmH{%G;8x z6Vfr*OGrR4C7}@aw~F}Q!ejo{VP42~JmeD91egaRvrEhxacYO_3Wy-sWm)srUb2I~ zrVB!sxkb(_#ug?5R%gkz z=g)msf{s*p>Q1ohygg#OPI0Nq(%uOMfwpPs99e4x3(F{@stH*6wJ+)f+$s!Fo==~s zC~;azUECoa9KKrCuCCP5ELGJFP_%sGb)RgxC~>TNn_vqi_+avjDYwCYHWQlypDs&Mbk+4JDXvzHgAZ2G#gX5k7IP>&9V zrmTF#73P{P+ag@K(Q?`>APe28Cn>0@{R_7$TtRVbt4A7G_O+rRQ|6<9VlQ8`ZTs^!_Z<7*M&vV<3#Ryv^VHg7i`*fZ2)=4e*##@nmRS2dZ%pw8FvQ)x#ozXa$pbE(POEC(!TMkGmmjM~yAnY>4Ay zU_|WcZ9Op;jqaQdXD!A}*;7_y(}p9&t=nxU-HyNCEr^QhLo?Ect-|LM9NUt6mJS z)f8ZzZi{CUU-U4(MF?>FJm5b9Y-W(}+%SCu#3`3|5A44S@HNdoor0Ls^Qta~1X(e= z`(5LcO$bBr5D8x0lh#=my*QvX#{?_(DPRU|f)=}&X?a1DgWLC}*0pK-*3?MpOIHY` z!qi5#d573JdhL98q(UU{k6tsrG2g{9UbHX4G?aEk>EwVqBP~3nF0h=oo~rfR5N-pw zfp1lS2Y@~~5s!xe)98)Vts%qq@9AbAHvvtLCw<#h9o&Aeo}t{^0}}1MshdqKY+eQr% zwUunkB||taQ|rf!EUptG{eTosQU5XpKRpC#EWyArX+Hd|qudrAKoxT{1)4@>3WjYSk8G}=6qL8VJ zz^9sk-pIF1&+F+5jJ}b8x-qNCsHf9kqryjN6S6FA>X7S#EJd5dKPQm`#uAcs7_Jx% zO<9P?HBbqrEz2xTyaqBg#%nauqhX~os}kG+8qs5ju|S6pRbfJD!5k6qq5qIGyBg`} z8UkiE7g0#sT4at(bg%Fg()j3ZYS0AM7;ZZmdiL+x{||u5{rXfocuz%zhv4Aw} zAly#JcpHMNRs13Y5O{xIyb3onDT-_bQ^F0$-rdyIQ#X3W`+{B>&dXcE;@a2Y=Y600 znsLwLGz9T+2&`zTSB7{|+Sj#EeuAPpbuC+$nAE$O`dR{sQk#gtA7i>qV-xScfc#-Twwc*f=1)`VM%Uy*>R`+eWrO+keG`o)DV=G2L$aHcog; z!6cj#%(J1{Hk+d(Y#D1~OI}Ilqp$q$?|#guG%_abzCGK3MtAPa+_~R#HGC5%quFvC ziH&JdEOw_G&2MzeA{>3emS&t3VjLD>kVT^?zKO?+NpZW9nr#?eGv4tv4j<=`~j4M7$L{dtLPw;XjD2(K4IO zyUlN+WE@WrUxEnF`i!r4F#IJrx;%e>aCUUsv%7N70D!~gtk4LW-#|;8MB;qV`#JdW z@;Dft{Mr-Wb^i9fel<8ad)Euz_YRJF1AhINPO~|hhM6FoQI@4yUSywm3WGtP!9K%+ zt)&rcp3jgR@#NlJ-6&0R070Bv2s4T@oMb!^rzbxW-k*De;Yt52IJ^8H_TP%mUFV0+ z-#UM(L+~ZKx2g09n*XYuN4N-!sm4?b6(^HaFJ_h|*Sy?WdK=`6C>jT|cpew-ntfnJ zJda!`O}Ys%Q<4VJBERvHjev;d7Fl|CZ(!){GLNz#3Fpza{CyGTIdkMH+U)(%=`>$$ ziOW19HtL^W1V8nDIX@TvcNp4K8#q zW^8Bmilu1Pw`n$Z$Xj_!A{b{V+zJl(dpS}}Jd1(qTZ=5bo`=CS3dd2_T>%KYR)Q@7 zyV?V~rO-<@Leny$j@T{!EeZ-T$>1@#T23ZW*1AdKF-)e&wuO8h%?kWYKPU5Oezh$a zkb4E~=KnSu@;&|lrjy;o7I)~vd!8P5QC$I2;(!S?VB1^NYhSkR?8DF9S(Jz!-LgpY zq}^;Zb$8m_@}hOXTB2v!XIj`iB>% zP28?H1x+U6bX3d~ZCP_p!(==Ia*`;{$BL4pSsW$Bf+<-;DG7rU$#3IgG!-qt&=$~T z6iS(b8GJYj&id!Cfkbk;+};cNKVjDr=%6>eI2c?EKqF%pXXPMn-8wW<`Fn=BJp1Xa z|M3ivL+1w|la?aMaJN}pM2ae`b^h~w2S0Gfy?n>-f^a-;aqG6wQ$cocZ1l^Q-6|uv zOG}os+1D+MXF}_$9kjB<2dx4K323-Dp9WIoQT|)2R0ICoe$1U9#9#_WQJx2+j^^-N zcpc@WGHAlc(aCbI-x%80jtd~UB@ zXOU<*IG()~XP2j^Bp;+&?6*7na~SmPTXuzp?}DTLS&u;*+Dc|o4inImQ9W^IcKIEO zM}yY$R(NzQyp%H|7&#-cxruKC{}12n?BfXd-W@3l^ZS4`2C6(g6!%G@JDvdPO=|$T zj)q0UESje%y4T-SnBCX)Ols!qkDwIlYUW?~rK($TNX+g(!g$ZaF z9OVx%psVJLGe90xx+r4M1ufgZx(Q+GZo_ODX=8B30tMUp3`A%&U$i)k0Ox`o-VFc^ zvo>#Z)fv8yivX}s;_KilO=m3{j)rJk3cW)U_YZ?Ej2guBu<$~MIJs_ZDu0Jy%^>S| z+y0Tk*CEqr6|>!9!Sti&aEdi#6T1RQJ_?nm1-o_!uYBE zgt1o)iecBPF=bfxHwBW=DLW>1(0lb zF~9`I$=S(8R|?KE9qN*t=qG@_gRbM{qIDgS0Jb*HrR=;G8*QSoaWN;Y&E*2@*7N-J zbCd)tm^K4+)kcyBkVgd=krppvTgR#(bA$7bM{L~W?15xOIGXYpLTi%8fT(|v5U%p4xnYQHP*Y2v_4#T-=LvLi-B(FME-Tk+mVXzfYg-9?cd8y8x^xJiyj80ukZwoH}|B0 z4-d`{F5b%(Q}>bsXq>nw;ypP#?kl|59k=JvZpQ~Frx06V-khbRX+qz!+p{k^5Q=)U z#_(!QJ#yOr7z{xukAl;a4<{GW82QFqPr(>`7bhQjesnlRDTr%Oq<;!HPy2@lC|WjA zK=Ve)O>5)b$wi>}T63M_fVg zeE2i#V+)z_#n!@gKIs1ggs9xrN`0zCK*{scnu(2*9ob@b0nHgdg)rLIC zv-k@hw?QbB0)jl9N5vHU)Qw#sK2Bq>4&ZbCH}%y(Z_xGFfBgpa*8bn$Ow+vBf3>^2 ziz$REoC6@zO#BNhPYKXK&`_^vc`x6Jzrb?Bf)_fPW-TPfi@(1hDM$L0-n^zaL@fO# zS-t^FqIn zZZ;}U*WKV*@ZA5y6R{TD;sF&xClYvMKm^+sx_@b0d|Mj#p)6`3y1@iotTz3P;&3 z9c%NEL$hIG%a)x(lOc^(x7V}uDxC40NQ1vS?f-ai8XQOnCf6IXY?R8GiuSJsXj379RNR^cj^1PVJKnB@Hea^l?hMNdZ zQ!i&&EVK9i0<6b}gMS2r-v31w$z<*ZFuq;a)}ZKLPqlq$zC+EmR| z0Le8#w5MIW3kia0CTv=#F~KhiTD>R3M;iMKKtYmqW~UL*O$3fdFVW(6H&3`%uQO5H+{aIy&n&7X z->(Fsvu7)kICLI{XJKU#McE1{rtA-n&^x{89bWW~Hrl4kvqct7;ycV%x+8CM=~Ygy z<9eBG&xwAi2&!QhJ~7)l$r~eLj?o;=qA&rAqH{Zs9gO;q6Pv!%L??ymMTqBwl|w^y zW@wHc6|x%MeByDU(lmC|p~z4=cf`ZDNft%s7*Ucy&K2HQh@ri~^>)k|-rhqi(NeQ+ zYt=S_xwS^y7f~?ANnwMI3k<+Fbe#iSZP^Qu>+*Ik`vWzOghJtq1KGSHxn;FayBhBK zydvd%mr_nOSyS+N!NydfAVS?nr3y~2CwodmGb?yAtG-;Up4V#mR+-m&F+o|j54HcG zus$M^w_N{FYYAjAMUjO`K8dni9-n*+!A9bNc}U`h8O;w&0i=Xl}elh5FoXi z-U^7ODS&W%NJ(TXD#N;&bkb7fXMU3pQW0u~GVMILs)o|dQmrakTUiF;pP*N*Y1VM- z9xU?6p>21(rZB{rsEydxv8wgKBBUZl#VXWk*@i?w*!V5o>c}|@>BbAvP?$iWhKdGR%{Y&=%R8F z^tPxd^tCWhiucFM<6}hOluoaW;_z~K(f=SFNF7?Q*B9#rk1x*-Ij8!ht{(>{7rn3S zdwG7^KRA-dPHR*8UGG9=X;tAKu{`h;mFyN_Ry?g3a4f_EO4^HL$g7e&Hl7=@nX?EaF4At4QcHE83=Uigk|3D8e&1OpXdzdo-r*8gN(TMc21IN-LP7)`$8E zM%v3sJOl4w3Q;aDojo6vYDwO-aE+J@oZC)Zu+|Kna$8)^wm zVztcJTm@Vj+IBhI!EgPRtX(tPty!FW!A$z5nG;$sE?|UqxqNCu5NN_5s>!i$%Kk7? zv^g5PYSVT{4-9Bb_K+rF30eN9MMGuB=8ea%so5#=H^G}~S?ecCF(ri6%*!Kdi=xO) z_^<0^~7FXR(YlqwUHbL%eD||FQ^=;trJ|i23Pqu4}MRIv(21l zE}})vc6roOjoPC^@l-pOer@=upZOrH@?L{ug?5w4p2vJsnpnMa+NiQ@xz>|Nl`WG$ z%X-jfJhKqM7iliFullqnixO&vrpXvhp1_jS$=GV3@^hypx*PGc`XFp#g(jHG!n1%p znnk06Qd{Y=kieEt#EH*Qy-c*QMYdH4;7>QCr&U)r$U&`I=8r9w(Ps4#b&wYEg0g@#KP!0S@~| zJ>*~l;9M*isdm?ib+uBnLh9C*`>w6>q-r8%pBGEdPBz89k~K_?g`ON5}5 zGRnyiXq-bcu+&%|y1Ox{o6oghm1U?RSKVq4y;cpcTeKH12FpaKFc$Mi;R3ATB8ydF zp2e6>uxruD^Sa!HJR{o?WTyB}xr2^N=ysN3#Wb z9q3)oQ;>~&42eKk%^4;zDivVp!rU2tn@ZkQ6peEjIW3|VeT_8LwYja{r>$<_Su8xHTrY2sB8^Gn3(89v8UF-mOr49+SP=I1_8yC}nieQw&vA+LvizHNi zlBv+fvLgah;W$hWp24w*5+oM=4*z`8jWCzb$a0cc9#k7tzw}W0l$drHIDCSWc8WQ! zz|6Sw1SnfN59?f zcB4H7R+X7@pHn+D?;D^vK$k#@sFP>Xe%4mp?_JjMLg z^WCld6%ggyZIlLGzg06-!-I)FAAr6uzh!#x>@oLRrAB$>wF_Ro+{c-(c0V>1U1-kR zHG_25+rTwh3l;UWB=$ES?q5a-Y0dg`L4m&1$kgwSf%weeco1_euZJACqV(a z{9vBrP>rLp=Fo0VoI=xv{F>YZvNVLQ%=W6W#%R|_TGWI>-m^o|tA_|5`-l@*4@hUF>P zS4?k}z?-8ZvJt39xlh$hl)MnzUf4E5qIOIU^;aD{rfgeds7?oB^k}usM+mNdF3&|9 zijq+&<_qjpF)XMF)ym2)&!+4|-pXzf%@?q9YG^k0GNzqi8AD5g@3LfursiL%Wj~)# z2?TF4O_3PH!c}Gz#e0qG6h8^$q=@7g23vC{*kqV}XiO+|QT{%wm01=9#0;Yd%HAMF zf+FSpau%tY&iQqfTO}6Dld3Q*)33rFa^r~!O8ymoF)G3trZw(m!R@#zbGtfe zfD?Y^k;=B&z2~`1>xU!EzhglBJjc5l+|z91O4qGs5*4E<22q{>N*Ah6OaP@<>~ov= zZWyoUB)>InHaJb@#%??uJnih;)%0ckKy=yy*ae z`Cr*T;F)xEs@SVl<$9>#ORDqICYKT^wAIKdc!i#uJ1O{OfNz308N&~zwly7<7N3yf$zadm-Qr2BQf z;|S5!J=CpY9Rk}ytJ#dQW&k7UCPQ}s!QG8TLVo(T^Z6^J&I(_Y+~G<|?(&T_ zh0XGGd?u=FpSo4k&Z6b2B)+*!yj3w$S10CU@kYpN`lNxAx7cBaPqF-*fbOSzasJ{p zg}w4j76*>ex_paiYIWD2xkt+~Uo`ql5o2UWyrjMsr~`!mzWW}acM%%DB2w|SV@EA$ z0M-T+)NH+=TS84eWNa%Fi}udq67i$ZzwUMI3KAJ;TX*ewXo=3Gq#)&>|Cw-pUJ(y5 z@UIUQmF8Dt&RHiHf#af-J1qIU@&a$MjQ*`N`uMQ3j{xM8JI7nnO?E+u5T#zGWQAXN zBIV%|uUr!Bmy1NFH~69U04^n4_jaf`&09mbuBW&>tyIIkQzuBVQX*uXh1`{?4O#f^anEi3#t@y?H5W%zgXx6IC z;^hF9ajta@>`L;Hf}XrD~9!yG7{JG2Qh|n0LQ=R zcCS$W$!;tDq)P*BmtRSkcKshUu??*btSCZ-vtC!1oLpxhoOL2Sc~?642GO+K+*T(; zwiK06XnAumsLVlavZf;ZoU|$7w9_^v4YF?bMh7Rf32ZLiIqFa-yh=0m^@5|8eWx2f zCFM@hm!7qI`A!U~YVrvvO4aZt=l=v8E0fFh-K-QdP}B2inXdFcnaCpVI44&Fmz+mv z+_H2o=We+s@e1G8bk=(mglulbl1BArZR-M6iSYD|{0SPZw{F{QGe zz_%Rvk~6e1z)4{EOAZ^}NF}3Rywd>>OvLk_KTHF z8LDelPt%(>>QZakX5^m&LXG)ucoW8Wk%my0ox88mDdM*$Y>Qum;V);0!SU(k@IBiAfE~U+*b6QfV+($?O>>a4 zYz+p)^iRquH20GR;w*cfZ(%kN0C(5AAAAMZXot^z;m=oilS7uT#~I5?rm{9siK+?P zKH&`!c%i`OuIo0EkN^;|YbAw2nhi^#oIm+N!Y7%ZATrL_=xX+_^g8Vd2!Cw$m<%hv z_qd-E<$g}Vcf|X8Bok1kOcAjpy81|GE~elQjnd_8ET-X2BqpFJKr+Tla%-{hcVia*b}z;;D2F1=?q%&~nrw6lK>1drby2)cuCl+G7v0zQ`dI?hL#H++L1&?QMxA@$Dsc%K|+J4lETF~ul!lUQkI z|k=vSv@R%Lons=q9Am6sOR_sPV~K1f6DfO`P!i>gXX;$fbv-{k_aE?H?$`DC<#XNF zeSIeZauX33uCN_rZe17YmJ9ZnOQR-b(jso>X11y44A-K%_Hg1{6jtvnnf)OB_SI0Q z^F9{l-NM4x6^U=;;(DcpY7#*4ss{3=K7T+s7{dY}wb0d-&wZE3_YETxlzS^(hUsivNdT7$~Joa*> z)A6cH#~t^!LNMuK%(jM$k?9k=&d8y|(U-(`OjY93gvyJyT?2Fb$ETNHjFvb2YzcsK2|_p zB&;RiqX#KF^O=da)bg<2dwu-=mOe)cJ4&!Te%y0+UgxDEB|{emxxeJNJ;JQCPTr5a zr;G_xGML)U3U=I}XUCSkAI=D%mgLRK%Ubcfr2lS12|S+=*1!mhkI5@m(RJ0sy3+SW zAt@Sr^^?>kK6cRJ>`2}%RIP5b)8xr*{y4A4iUdFM){1(iL2@_hoMAr_we&)I*T0T2 zD{yT&@GNn6C=Q=EUDMLvpff|UD9sPrp#{(zHmjOW5!d5veb{Z8_ZP=#{6VMq&8 zMy5)UHgzbM^eSd1WqrG&rbYOZlB9uf`~+1On7{AAW6QKNOl5cj-DJ`_e_>bWsiI3R z-clN>3X{rqPb@)=)Yr!6C0m)BC|J<%z7`pQ@wm6Keto ztR|MFU513+W7fAva?=LB-Q;(rPaNniQK*W{_;~roI_0~Zv+2SHC7|cTNL8kHW4mfp z_`N&#?Om7TS1vY}*-u*aGKp16p*Vl^WgHP585MRk zG6?48@`uuC!ABj%Z!xXa*U$Fhl>*RCtQn|?^iNO6 zUap46gh&##wwi*22C0ywcq|+U)N?#1StbkNhV|Tif7Kkx3Y|X#mP_0M;^F!4u85R5 z?_Tp6bNci#?_0~se8@yn69L2+H<8^B7ceq0J1y2O{$G`v0&1XY=?yL*OY@E?%SwK> z3P0#5fc2D&p?$zX^+y0WU|)I^#+>XFn^jm`c{V9=W%31Z64BA{)hY&b+$Y2nph zI#^*N8Z^c2f{RbnVMtK5QtLeQbvQ?!i1|fF2gV-cmX*ox?RL7{zWyGaXW}Rd^61t8OKB#`n(>qMRN|7|37fU))__V1*sGfed=b{l z&~zwt^!X@Cl^jb^ka=S^0Qu^va6wI&b6f!TeDkv&szV%3arIfB63)NUTjByJW4H?5 zIW7bgJ+ywS{?vV>?8+tB=moh*G2TKSh;}>{0Xm=D1wxBtBP`E9LMo{Ot?dR#*Q9*D zD9HB|91f-%V}V61WBkq39JXPYCnC@0FT8n7_#`-K><@;R8mV>3r=N__3>ar9;lywepf)gPvJ!EyF^r=|zP^i1*#JCC6!r2%Co1T?Hxi7!Y zJdswbYMBap-Q;zd4I%BGBLGCWPwCL_sve&UJn*N520AmeMhMUcM-&(E=?Pl4#8xh3 zlPoZ(d$^IBQoU9V#X0N4&HS%Q3a7ky(vq>Kk}WcKUrbQ0v5vzUi^0fq)>=9`=zyq5 z96p?Ih4^DoPAe96v+!i?*3w4J*uS1kSSPhsf#z<$fX|e7cCp~zr(ZgiJoyrz0l}xt z+rY3QbKk9%p5xogBQ_y-m<@eu+_zom({|WoUVfZQ?*u}7f5+vWhU9J-kh9d#qCuZF+<@B z2!`}|V>AHTPDsDyXyZw#vXYzcQR{|dv&u|P7#qI-D_`|ct!)l|JMd@3h(1n40tu_N zK~X1s{3X;026Nowi(H2%u$@SG-!u1>ZuA6awp+QE=X!S1u66V;*KT|trmL&9H>7<> z55grdLSGkn;Z0ijZr$5L?(Wj5CGm2gfx<%BemZz$^pMP-u2l)d- z?}!P9yxdA)7=xt3<>Q;a;_@8wCj7uwbYLdzAG*=Lr(r`ML49Sti|7jtAZ^5f;j0OJ z$_3cdWN)P37>?IMlQAp+X~^rqfelpcJXa*+c@fTgD&!qfVV9Y{1>qG;4+4BPwiyC) s8|!*~4Xa@^J8%hCPWstmhwg#c9Q>8h7EoFYxtCroOBv5K!MKF~2i=v?SO5S3 literal 0 HcmV?d00001 diff --git a/test/lib.py b/test/lib.py index 723958ca1..ef08c2032 100644 --- a/test/lib.py +++ b/test/lib.py @@ -10,6 +10,7 @@ from array import array from cStringIO import StringIO +import glob import unittest import tempfile import shutil @@ -46,10 +47,46 @@ def wrapper(self): return wrapper +def with_packs(func): + """Function that provides a path into which the packs for testing should be + copied. Will pass on the path to the actual function afterwards""" + def wrapper(self, path): + src_pack_glob = fixture_path('packs/*') + copy_files_globbed(src_pack_glob, path, hard_link_ok=True) + return func(self, path) + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper + #} END decorators #{ Routines +def fixture_path(relapath=''): + """:return: absolute path into the fixture directory + :param relapath: relative path into the fixtures directory, or '' + to obtain the fixture directory itself""" + return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) + +def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): + """Copy all files found according to the given source glob into the target directory + :param hard_link_ok: if True, hard links will be created if possible. Otherwise + the files will be copied""" + for src_file in glob.glob(source_glob): + if hard_link_ok and hasattr(os, 'link'): + target = os.path.join(target_dir, os.path.basename(src_file)) + try: + os.link(src_file, target) + except OSError: + shutil.copy(src_file, target_dir) + # END handle cross device links ( and resulting failure ) + else: + shutil.copy(src_file, target_dir) + # END try hard link + # END for each file to copy + + def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" diff --git a/test/test_pack.py b/test/test_pack.py new file mode 100644 index 000000000..0b9e448cb --- /dev/null +++ b/test/test_pack.py @@ -0,0 +1,16 @@ +"""Test everything about packs reading and writing""" + +from lib import ( + TestBase, + with_rw_directory, + with_packs + ) + + +class TestPack(TestBase): + + @with_rw_directory + @with_packs + def test_reading(self, pack_dir): + # initialze a pack file for reading + pass From 099ec0dbd23bf46cba7618768e628ceeac7c2e17 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 15:17:29 +0200 Subject: [PATCH 0012/1392] index reading from V2 index files implemeneted and tested. Added LazyMixin type from git-python --- fun.py | 1 + pack.py | 190 +++++++++++++++++++++++++++++++++++++++++++ stream.py | 4 +- test/db/lib.py | 4 +- test/db/test_pack.py | 2 +- test/lib.py | 2 +- test/test_pack.py | 38 +++++++-- util.py | 69 ++++++++++++++++ 8 files changed, 297 insertions(+), 13 deletions(-) diff --git a/fun.py b/fun.py index c766f8e09..883062eba 100644 --- a/fun.py +++ b/fun.py @@ -113,3 +113,4 @@ def stream_copy(read, write, size, chunk_size): #} END routines + diff --git a/pack.py b/pack.py index 676fa26c5..a175f48dc 100644 --- a/pack.py +++ b/pack.py @@ -1 +1,191 @@ """Contains PackIndex and PackFile implementations""" +from util import ( + LockedFD, + LazyMixin, + file_contents_ro, + unpack_from + ) + +from struct import ( + pack, + ) + +__all__ = ('PackIndex', 'Pack') + + +class PackIndex(LazyMixin): + """A pack index provides offsets into the corresponding pack, allowing to find + locations for offsets faster.""" + + # Dont use slots as we dynamically bind functions for each version, need a dict for this + # The slots you see here are just to keep track of our instance variables + # __slots__ = ('_indexpath', '_fanout_table', '_data', '_version', + # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') + + # used in v2 indices + _sha_list_offset = 8 + 1024 + + def __init__(self, indexpath): + super(PackIndex, self).__init__() + self._indexpath = indexpath + + def _set_cache_(self, attr): + if attr == "_packfile_checksum": + self._packfile_checksum = self._data[-40:-20] + elif attr == "_packfile_checksum": + self._packfile_checksum = self._data[-20:] + elif attr == "_data": + lfd = LockedFD(self._indexpath) + fd = lfd.open() + self._data = file_contents_ro(fd) + lfd.rollback() + else: + # now its time to initialize everything - if we are here, someone wants + # to access the fanout table or related properties + + # CHECK VERSION + self._version = (self._data[:4] == '\377tOc' and 2) or 1 + if self._version == 2: + version_id = unpack_from(">L", self._data, 4)[0] + assert version_id == self._version, "Unsupported index version: %i" % version_id + # END assert version + + # SETUP FUNCTIONS + # setup our functions according to the actual version + for fname in ('entry', 'offset', 'sha', 'crc'): + setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) + # END for each function to initialize + + + # INITIALIZE DATA + # byte offset is 8 if version is 2, 0 otherwise + self._initialize() + # END handle attributes + + + #{ Access V1 + + def _entry_v1(self, i): + """:return: tuple(offset, binsha)""" + return unpack_from(">L20s", self._data, 1024 + i*24)[0] + + def _offset_v1(self, i): + """see ``_offset_v2``""" + return unpack_from(">L", self._data, 1024 + i*24)[0] + + def _sha_v1(self, i): + """see ``_sha_v2``""" + base = 1024 + i*24 + return self._data[base:base+20] + + def _crc_v1(self, i): + """unsupported""" + return 0 + + #} END access V1 + + #{ Access V2 + def _entry_v2(self, i): + """:return: tuple(offset, binsha, crc)""" + return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) + + def _offset_v2(self, i): + """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only + be returned if the pack is larger than 4 GiB, or 2^32""" + offset = unpack_from(">L", self._data, self._pack_offset + i * 4)[0] + + # if the high-bit is set, this indicates that we have to lookup the offset + # in the 64 bit region of the file. The current offset ( lower 31 bits ) + # are the index into it + if offset & 0x80000000: + offset = unpack_from(">Q", self._data, self._pack_64_offset + (self.offset & ~0x80000000) * 8)[0] + # END handle 64 bit offset + + return offset + + def _sha_v2(self, i): + """:return: sha at the given index of this file index instance""" + base = self._sha_list_offset + i * 20 + return self._data[base:base+20] + + def _crc_v2(self, i): + """:return: 4 bytes crc for the object at index i""" + return unpack_from(">L", self._data, self._crc_list_offset + i * 4)[0] + + #} END access V2 + + #{ Initialization + + def _initialize(self): + """initialize base data""" + self._fanout_table = self._read_fanout((self._version == 2) * 8) + + if self._version == 2: + self._crc_list_offset = self._sha_list_offset + self.size * 20 + self._pack_offset = self._crc_list_offset + self.size * 4 + self._pack_64_offset = self._pack_offset + self.size * 4 + # END setup base + + def _read_fanout(self, byte_offset): + """Generate a fanout table from our data""" + d = self._data + out = list() + append = out.append + for i in range(256): + append(unpack_from('>L', d, byte_offset + i*4)[0]) + # END for each entry + return out + + #} END initialization + + #{ Properties + @property + def version(self): + return self._version + + @property + def size(self): + """:return: amount of objects referred to by this index""" + return self._fanout_table[255] + + @property + def packfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of the pack file""" + return self._data[-40:-20] + + @property + def indexfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of this index file""" + return self._data[-20:] + + def sha_to_index(self, sha): + """ + :return: index usable with the ``offset`` or ``entry`` method, or None + if the sha was not found in this pack index + :param sha: 20 byte sha to lookup""" + first_byte = ord(sha[0]) + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte-1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + # bisect until we have the sha + while lo < hi: + mid = (lo + hi) / 2 + c = cmp(sha, self.sha(mid)) + if c < 0: + hi = mid + elif not c: + return mid + else: + lo = mid + # END handle midpoint + # END bisect + return None + + #} END properties + + +class Pack(LazyMixin): + """A pack is a file written according to the Version 2 for git packs""" + diff --git a/stream.py b/stream.py index 10bc8901a..44c7b945a 100644 --- a/stream.py +++ b/stream.py @@ -361,7 +361,9 @@ def read(self, size=-1): if win_size < 8: self._cwe = self._cws + 8 # END adjust winsize - indata = self._m[self._cws:self._cwe] # another copy ... :( + + # takes a slice, but doesn't copy the data, it says ... + indata = buffer(self._m, self._cws, self._cwe - self._cws) # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) diff --git a/test/db/lib.py b/test/db/lib.py index 738eeb851..35823059b 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -1,7 +1,7 @@ """Base classes for object db testing""" from gitdb.test.lib import ( with_rw_directory, - with_packs, + with_packs_rw, ZippedStoreShaWriter, TestBase ) @@ -20,7 +20,7 @@ from cStringIO import StringIO -__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs' ) +__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw' ) class TestDBBase(TestBase): """Base class providing testing routines on databases""" diff --git a/test/db/test_pack.py b/test/db/test_pack.py index 29b348876..6faff4695 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -4,7 +4,7 @@ class TestPackDB(TestDBBase): @with_rw_directory - @with_packs + @with_packs_rw def test_writing(self, path): ldb = PackedDB(path) # TODO diff --git a/test/lib.py b/test/lib.py index ef08c2032..6b25876d6 100644 --- a/test/lib.py +++ b/test/lib.py @@ -47,7 +47,7 @@ def wrapper(self): return wrapper -def with_packs(func): +def with_packs_rw(func): """Function that provides a path into which the packs for testing should be copied. Will pass on the path to the actual function afterwards""" def wrapper(self, path): diff --git a/test/test_pack.py b/test/test_pack.py index 0b9e448cb..a0c8464e0 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -1,16 +1,38 @@ """Test everything about packs reading and writing""" - from lib import ( TestBase, with_rw_directory, - with_packs + with_packs_rw, + fixture_path ) - +from gitdb.pack import ( + PackIndex + ) +import os + class TestPack(TestBase): - @with_rw_directory - @with_packs - def test_reading(self, pack_dir): - # initialze a pack file for reading - pass + def test_pack_index(self): + # read v2 index information + index_file = fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx') + index = PackIndex(index_file) + + assert index.packfile_checksum != index.indexfile_checksum + assert index.version == 2 + assert index.size == 30 + + # get all data of all objects + for oidx in xrange(index.size): + sha = index.sha(oidx) + assert oidx == index.sha_to_index(sha) + + entry = index.entry(oidx) + assert len(entry) == 3 + + assert entry[0] == index.offset(oidx) + assert entry[1] == sha + assert entry[2] == index.crc(oidx) + # END for each object index in indexfile + + diff --git a/util.py b/util.py index 291855630..5c2bb540b 100644 --- a/util.py +++ b/util.py @@ -1,7 +1,9 @@ import binascii import os +import mmap import sys import errno +import cStringIO try: import async.mod.zlib as zlib @@ -16,6 +18,22 @@ except ImportError: import sha +try: + from struct import unpack_from +except ImportError: + from struct import unpack, calcsize + __calcsize_cache = dict() + def unpack_from(fmt, data, offset=0): + try: + size = __calcsize_cache[fmt] + except KeyError: + size = calcsize(fmt) + __calcsize_cache[fmt] = size + # END exception handling + return unpack(fmt, data[offset : offset + size]) + # END own unpack_from implementation + + #{ Globals # A pool distributing tasks, initially with zero threads, hence everything @@ -76,6 +94,28 @@ def stream_copy(source, destination, chunk_size=512*1024): # END reading output stream return br +def file_contents_ro(fd, stream=False, allow_mmap=True): + """:return: read-only contents of the file represented by the file descriptor fd + :param fd: file descriptor opened for reading + :param stream: if False, random access is provided, otherwise the stream interface + is provided. + :param allow_mmap: if True, its allowed to map the contents into memory, which + allows large files to be handled and accessed efficiently. The file-descriptor + will change its position if this is False""" + try: + if allow_mmap: + # supports stream and random access + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + except OSError: + pass + # END exception handling + + # read manully + contents = os.read(fd, os.fstat(fd).st_size) + if stream: + return cStringIO.StringIO(contents) + return contents + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: @@ -93,6 +133,35 @@ def to_bin_sha(sha): #{ Utilities +class LazyMixin(object): + """ + Base class providing an interface to lazily retrieve attribute values upon + first access. If slots are used, memory will only be reserved once the attribute + is actually accessed and retrieved the first time. All future accesses will + return the cached value as stored in the Instance's dict or slot. + """ + __slots__ = tuple() + + def __getattr__(self, attr): + """ + Whenever an attribute is requested that we do not know, we allow it + to be created and set. Next time the same attribute is reqeusted, it is simply + returned from our dict/slots. + """ + self._set_cache_(attr) + # will raise in case the cache was not created + return object.__getattribute__(self, attr) + + def _set_cache_(self, attr): + """ This method should be overridden in the derived class. + It should check whether the attribute named by attr can be created + and cached. Do nothing if you do not know the attribute or call your subclass + + The derived class may create as many additional attributes as it deems + necessary in case a git command returns more information than represented + in the single attribute.""" + pass + class FDStreamWrapper(object): """A simple wrapper providing the most basic functions on a file descriptor From 0717775fa51460335976a046072cf5c492af2a91 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 16:18:04 +0200 Subject: [PATCH 0013/1392] index: added tests for reading version 1 index files --- pack.py | 6 ++--- ...438c19fb16422b6bbcce24387b3264416d485b.idx | Bin 0 -> 2672 bytes ...38c19fb16422b6bbcce24387b3264416d485b.pack | Bin 0 -> 49113 bytes test/test_pack.py | 23 ++++++++++++------ 4 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx create mode 100644 test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack diff --git a/pack.py b/pack.py index a175f48dc..2ffc64f52 100644 --- a/pack.py +++ b/pack.py @@ -66,8 +66,8 @@ def _set_cache_(self, attr): #{ Access V1 def _entry_v1(self, i): - """:return: tuple(offset, binsha)""" - return unpack_from(">L20s", self._data, 1024 + i*24)[0] + """:return: tuple(offset, binsha, 0)""" + return unpack_from(">L20s", self._data, 1024 + i*24) + (0, ) def _offset_v1(self, i): """see ``_offset_v2``""" @@ -75,7 +75,7 @@ def _offset_v1(self, i): def _sha_v1(self, i): """see ``_sha_v2``""" - base = 1024 + i*24 + base = 1024 + (i*24)+4 return self._data[base:base+20] def _crc_v1(self, i): diff --git a/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx b/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx new file mode 100644 index 0000000000000000000000000000000000000000..87c635f48cbdd3ef50ed7f14bf339382f9af6e5a GIT binary patch literal 2672 zcmbW(2{hE(9{}(xgBd(K*&dTMYnGWTWyziz99xMPjlC>~p})toMV67`B}1}TmM2~q zOyNC)B$V=>Y~wFNCXZJ=WU2ml&dE8R)9e5G&pDrSzu$ZB%$eWs-0!^?fWLg$V76@! zjQ4fx<(7XHr!VEZiu;rwkvFy2WR<}9M0 ziNW?u#9{x}NWk%Jk}&=fDcHZ8H0;~0S!1ubC&}!{~9egzD*m(EIK>r!k7i~*+JOuqz7{r{l8%V`*$;h zeTQ~A4D+ushU0(71dg$o?qUY>ZRRlkf+Roov-HpfiknPsQSgc&e(qq6WB`1JWKPan zB@S2bs@9j0Pu|~d3zptAo6i9t?h^kI1DSrnq$x0?CZnAHC2c>5vI8K!Ubt5DVi7iY z?A_8z*$9G-R#3e*0)XYZY(?$$*T}c}X^F@3#WAThDSD(2@KOCVt`3W24I`2RsnLBe zS3jw0nq2P#z07L^-wp$TW4myC&t@7Gr+ZVVr6g z$-e-=Eo<#hN2Q21t3Ra-jQb!Z=9wbXoA5o|tkXGKDP`nmyOsVhEBFSzNeHEyv<^U> z48}jq^E#hMi98}g93TD4@T8`>B>;(nSoROf%FTl_#jluKzhpcYt}lD|3V>@9rjgfirN;=VrfBvag&LO4B?q530k~0*+caMbZa7Y`tgx)%5tzpQd;u;Cr29|` zg4`9@hf42=_8lvAG#>GT`&3S$x6E;z*~pNXEd#Z#RO_xjl{W4P*G`Rh8+*|#F9jD) zzzDbvYWkWWLN)=2+6tU8v~Iol{rb;BG-u|EgvR$DD^I}Xt$J>yH>2(npZ;`;ap+T_ znF0E536B7_Gth=y2XN&VQmRuEmkqQ=~}@d@(%C z18d}t(ht1#T;EwyGmkzOP?_LDh^KKd&j66;5L;?~8AqfzuAk{?`@lec{4O#9*1v>p znw+nYlJ7^=)DV*V%wjyr>I#OB1M8oyQMA%DR=eLa4S{}gSy0&{e^`fw-dVq=QcpE!H4pW15dKd_d;q9mTz)fwH2@cAKACp2 z(fn9~Q7%3ZfN13_?p7)l=hMSHgo$4NJ~#DHajXZ@8?8!Uy7Gtdti%pQ4XVV&guHY^Hum*BmVAK=#l8yatThW*Cw-l zA0*q7;d9|z3{TR1LV}0_<6emXY1JSW9tQuv_t39)r0UeSghr*wy)K~ytmL%wom}|) zzBI=lB7L31T2D1nQDEV>+91(#F93cD>|VO{CQDY@>+jHY6ZJI9(SGOU(!DkXfXl`HYn@z}jw8u= z{7#8S{PZ4&STf<649oXra}3>5Q?omL(>2z%3 z72f@_1ptxy)RCe1UQ}tc#qSXAaBd2{;EHe_0IhK&fSj*j!TtDkOh!-c0d><`T|+AX zP;?oJ-3H6|xa-CHDhA4iboaYr7McJkX_{=;A~u!CHkB56{}I1>PLX%QJ_%k6a;NVw z@bp@dquPOdQ7(IQPVPbQK*O@|lhkvb#-aP;sLT0jd86!ebCf6}!;r2C_hk1|YUt&NPYZ&3 RQ*{1wQC`jG0&mm7{{mdj0G9v& literal 0 HcmV?d00001 diff --git a/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack b/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack new file mode 100644 index 0000000000000000000000000000000000000000..a69b28ac68bd547d49ad7502b1b3fdd42d74ec71 GIT binary patch literal 49113 zcmV(^K-IrcK|@Ob00062002Xo4tSiM&CLx0ArL^}J5_KuF~jh;B*xf-HUPumk}Qis zJlMTkgV&dQd2=w!;;H1s42lU+BavHz@?cHyOVZwJGR7}TyyD<}N~cl=9hfapawdCp zS{ns65Fc}d^V_uBgH*<#(!)Z0}m^v=PtMR@`Q z5JzowPX5;mTI2rYEKA*3`~cm7Mv$8ec$}TgO$q`r3_#(1Pm#SK>9mtpkik3X1yUzE zf&M_P3vcg$H*ovld!vg$ijL>F5_uBh;4Q9ceng9i(Nl7k!W59v@Ox~n!&MT$hH{15 zrjp^DKD|{f?eZ+F`FL+-0XqdEJXuF@zzNapwdfe~uQytCM;U%2^99@ZIe)K_0eGBE z)q6ZtX&(S^ZNkVVx0dwEODOADY9`rg$Zd6T`5>(l&YU?jbC{Vk@0l~o@)njQtX;j? zP|0n3UDmSemT6_zL=tVvEBB8Gnc8f%Xi&Z9FlW~P-_P^>ZqM&=ba2>Br<;Qq#6zNC zd!E{{tjS~31uf zA1Mt4LOdA2F_?57#dHY^&~_Z)A3~Py{)~= zrH)kw6#|feEhgZJ?qv0xTSy1(TyqROP#D1M;}}s;ED-=4WI03Zj`vx={-uhr50-$o zs)u*aUS1oiWNa8hF%T!!){@Fg&$M3cNliGMYfMk6NOnqpt5SR%B#3M0Y~ge`tIxGS z!^TgvZNc3S^14$Qk z0}~E#oPHO2$_-SCtP=3@kV&Y@7u(30zD6;4-cOT`%%OZEfs3oIl z3}Nn+mVRt?*)qw+wm>Pk2o4Zc7lAk+!4W=zfAvi-Pk9B^T915m6onX@$AW&$rZ5JI z$l;XNjt@w4^mpv{RSRX_+^#4)<#A6%U=bl61PoifF|zKz&e`j;FFUz6?^}b5O2Grg z{ZSzCGc!qvX^%QK$hP{y+Zcoy{X9rb>Yb zaTpV!{FT<8ay`QxDm!yM0B_5;$C1Z9J?r->As-Rr0LqGlA)K5j-2={MooaH~HhJ_& zbkON7e)nj;l;i^f7~_)X1*W_bBy(V_eQRUWaBuwTQ~An1&m4+M#;=E`? z$c9f7R2J}s1P&549gwGqiQCU4)^^4h}KJ_)Syz==z6(xreS$?wZw>$K-ANtJb7l#}&il~vT zvu{}QH{$_hKM2-gcca{=vw2sDQJC9#$JgOUTjxh97KIZ^NY>-ZT;SYn)s%UP#^;Y{ zdxx8uoSRjY!cbz;#N?Hr*%2SNzV1zjhG`1x$`f{@se@rk^)Cs!JRq$vuD#3z7Me6o z#gqPV^p=LY@vBOL!NhU{S<8#WRbsrc-Y>oRvNlA=3TA8`FuT83F%H@KQy`zLTU??0 zH$HfMfP&T5^mbV5-G`ER;BQC@6ODbnK& znYB>x&K6_$5OwbTKDNq4BVwZRgmU~d!!`KldDGygSoaJ5gHm6K?lh$&Lh|hjKq0v< zMs7Obbp>1N3iH7~&(86!bcXyAl~4=|+0?f9=H(E>S?x<1JHvM(MwbEY&yGMS#seJz zK8i+3$kE+-u+(%v=g`u*JDGwjy>p2#8(M@r!sOG56}h36Jo{gW2A1zj~W2KQ6= zVN8^IbvQ^~^@UEg=7(O=jDwFh?ikWkH%?tAXQ=?WR_R<(*rg4I1(xb!5!1sz@yy$zl zZabS7BeuQ%X-UOkm>LH+J#DY>yo{{wU)cK>rpg*+_D&|N7$JtOTAgQZXjdC|lynt! z@7f+rpRQ91iE#Pdl8}t-!UWg)xZ+x#!*J-aBVt$=&9m{rj`n zw6^R|>|C$i>Yh~!G9H+$M4Dl~o|f;6TmGge>`q!tSfkznQ`VfGgOuuOK^I*^SpP;7<8l4T>*8AK@jT28Qowra0&H{xa2 zw#sqai1#PR7#v&^guG6*$cdC*Wtz3micVY#WIQwq&byfEb@VTK#c^RnttU9l7h)}^eh+r zmvU;+MslWA_=SAeRYPJT$ZTm4S%Vt%iAE1%1m8sJW6q7REX!bct~!c*myMBsdCoD2 z7LxesP`2?&@`P}7+~yoNLxhhTb@hgZ`(-}$G&R?`Cr?CrGyC2W`?_FIKoxv%>&~=- z2ALRCtt4!P{E2#J-w|jX>l}8<4J{0zw+HSj>Q(js9TkQ&o`Vn_@9X@4CItSN@>)Gp z$=xY;qBiL^5nY^gAG?C&bhh0NSAhD^Lbv^a(s6&N=)#~if zl~tXsi}TIr75@iasJt$)GkBbRQ^9V+FbqA@udu`cT2Ug*<=sFkj_%!DGVkN%oPadjLbWSAT0=J zwNwrbI1R3*jBdqShhUxQ3fCEgXc8O)*h!ah^^{u$5W&4XAT?yLRJ#}qQiY;E$k;QH z&R>w~T`e4!71|iItyZOFTIueB8!Ql=PUAu=>!7FwC1W7Hp_c%UtXzo(I_q`f2gh&$ zKefm*YK~9O(|<9QXsicu^QAV>c6lQU7FpC-yq=-QkZ#d6&NB7YtdmW;7#~6y^Jg~e zsR*yZ%lT^i2ioiD!PE72kzB&p>1q|89ND5OK@*n5@3q0cV!8=ueBehL#voM5r6`IT zMhDS!ID$Ryb6yH;8Vm2_8@Xh`|D?5HzaUi>o#&#hIXaIe2^>M8R zmqvTCY4AmEkmEhsm%vuI+HXY+w+SO&mmy6?R={t-KL^(L4JwolqPG)xoMn*94uU`o zMc4K#8rUcy8rSYzxiIk)2+VY(iXAc!<0JmNfEc5$+S}fqHci=xCWGtkZe463?Q#}+ zOGsH%6E$ICs{^SpuM+^e1agAPiVvvj(qUFLwkpE4r4@h=E~Qq@U*ssB_d!w%&wx#nlc#A3;LvH%T;y|Yy&z5y}+0f5Fw8qLF zJ>Tdb_D|_ODT!u-i){P~W(#-%_-SZ)xiWa1ZIZ!m+b|4<&-GIf$i)d%_W^e5+O)ZJ zhrEEH$doN4rT|J#nzx@QIdih@V$i>+kNhdFoTUye{M^6(dfTo) zci=*%l@l-%Gg3TVA%V;hJ7FZY0@5Sik{48#6@;icwht*1dn>UZ@Y{vl{Cq7AI{u9c zgP%@aVS!~oDsgSfebR?hvfJ` zvNlIBdJ6cD$>`yb+c2K2>nFxev=O<-oXa@+h!5r-N1_WqwJMl7Pui%RP35oarjkcH zd)Lh!w~8VLa=rYp;_oKU@Gcb$NgZvySgexd z;;MQ5`o*-Pep)S8o5kH?y;-h)V~lzJRnOntEf(wR+eHw3*)!JfK74NVeLwjA_9MT& zzWY5eewec7DJ!&L(&|z;$XO;V*cr=85Q>SyfnkjWmOk%Py!r5HHNU?55>))y>wf!g z`F_rWVbA-i=Uva|i+P0e^E5%LAB6$sEEA;xksnwsj79rG;r~2`{MVt)^_Sb*#pZ7L zmcL)zY=W9!rjvpD995u=p+CA{cCQ~*)MSUsN8(%Es9cx>@2q$|Z-kW@BdDel zPHN>9WLhDhY%dH;YlA`gil82}%<0-F;JI)DNsY2{r!bCEIR?U%2d*1eH!f=581Vqc zijozj*adJynIs2g;nmgUBuNslIi8i~6__2^{{AS+`*z@33k z%F3!PWpUtpjITC_NJf&5N2hDewaLM>1hL~GZmkkhctqNkdV55Id($Gk4hqMO-0fZG^P@onDC+B=3n3yn zqG2gw3fjPCdoWl8wanUJ7Sh;OhHF(~lw$VgvM8QR9P(NPOP1-ICiW1_Dd9$ERAC(1{0u-pS_wMU2fjKZVm|E= zk1%7Os>}y?#Ij8ogWVhpzG(E3a@7)iv5*=o|AMwyWJw!odx+DOLc=2R;h^0asJX6P zlKNBO#kwZ4b&2ygjv)M2)ccVzrXUjiPn5GYWD*CbA0)O*w!N1%7G$Gb0M07$wYzso zFE#EO;vmpdx>SMg?NE<~cA&*v3x^ZJ`n?$MdmAv&`WnC{Ir14-hV>Ub&(3pcu*c2S zt;d2wO(Q*)TBL+>`+GJSK3+IIJGg&$Bri@GtHtLexjLoKmp3=~mRW6*my-#4iY*{5 z0{Si^Kir`PdN28{c9Ht@4>uSu6nE2$<9(i9vU96@I-5MulSY=n&Z{~lqmx+DU{t^$ z6>4iS+Ec@LvX|_1Go$IK#@7e9rXHvvnlQU;ESVzdq4VV5Y+#7S2KHDQC3F_>b+Nk` z3gG=I#U$?6i9clm_stflTR@2lrhPn!jn2_d>J3q`oR3NmaFiPCC+H*kr06IKl`!}1 z{^V4($O22Xe{rNc{Pw1MasOaAD@z=f(%)(SR3zUy^%=btPiSdV$3}+|DvqI=jn@Z} zL&pq(&=d{Z<6`s}WGVjA%3|T)xtC&EpMEd^M1Kb#46z)gu|pD{u1AQD%kB)&|D~Na z0(ZK#a;n}7`OOzvR%HI6`)$&0z_2|pyF0aNf5QipHp2*?(bc^)1NlE(JgK3{Gs0d> z{sFYLl92F@0Wt85KvuH%8&3E!lmCbI{Fc5%m(57^ug>pea)Lz=E z<;JNuBoLJERDxsuK^bge;~qA47! zLAA%>>g0k52F^Ntt<6%iTNFJ`Or7F+L%y}02G(+M4FxmvpCG(DmfBBZc*rgAf+@@! zTS{Lu-KpcX!g*JoMF}T=!CC(4^6F!LT1o{cA%ML}16pRhDDGZnZ|?qX=TG8UYvZ|x z0d7aww(4deJCRsX%LSF1E8dJGi^Os*YKl5tC?F;U_bj+aVGZKibbf&QDa5iAuo|vr z;}zrH+X26ru!WM9#x~p&U+~h}F|MZ*VyzU{BsQUph7AH-Q+3!1(o=O2F>hh~`PrIx z+V`csJILOSVHekRdxvyqZSYz_GS->C=Dvyx-;!m2galdwwgQ^jT@ ziQPyQM9jP~YT$d3H#j3RAM+D#3NvC zxgF9h3-Whmp(P;9AqlPazNHbz=9U_j74v~T4osPf{EFugM}p(8%8^11EKUDz&nz^? z)245@RAJ)~(Ewemrp}7?*D0QJNA<3;=`3lLHBD#i`GhTq4j1P%%rVo=f_SEJv{7O@q+=Ib+VhI|6DBFmavbMQKTzeUw`&W0CmvYE{Y7GKe~p#l_g;=! za)k~MjA@nSP3q@PkHb4pvb3zj^5D9fdMT<{T8XaQ+kU;~hf1-6Pr z5z19~RgbX1vdsPg{U{?~vnP0*jgdW1!!Qtr=lEBgl%+%&fHfNei2<=OAr!gxxv>;3 z2PZ;uptn-vF%88kx}^xgaY?T1LJInj{*M2U9LWMQXv-S6aO zB0FL~pti^vcIs%&$oQ3f@Dangi+qs=GP#(!*)ubWW%ms7x^PIR@fl$)k*s&>2WHpWBDzI*oQ+dWZ`&{oJ;%R-XfBD7q8~f@pjt72X{EAKYVdv4XFPvS@=7Mqlnfss*<824q8Hz1CX}2%6 zzb9!}lvXjUl(mW$LO=B-tc@@C!wVTAm8wd918Ul!U zG#cHF2K+-X4ShKkXYI3h9p~NF5AY=jW$}IMhamLknJ>lD>mc3rqQ%lpce6B@@6DcS zl}Q>;vZ+}%bCVgCltrU=?L+5Xr+wZTmW4n6J`W~)vo{Dg72Lny><^st&PA_1?l`A| ztNwXQeETN;r`J7oe(enVSC`^vad3BdbZ}6R48P`XDo>*<^qbyJ(h{QCgr9$bU;O*0 zmU#0O?c1>PR{PfLzt_6R8#fO0 zLTKM;9O@;2qtQ6hO8`%!@w|pwog&4H{cVwjH(|62#Ug?UYX*yDlqA7qE(Hk0BnaI! zh(a-Px)p;*DZxFI%9V>e82RX~NC;sCMl!Du&G zN-vlO9%I`R-IQ68MA)og2sR?>CXs&+gt@UyO7bT_O)QsCCc~6u02>F)gsXn{lUPMD zjHHQ_0ZM2crZ63=z=yd^qiAohwPj1m+wQp61?g0Csn?BEU2}n}3VwA#}HpYKKS^&8{|&e<}8 zwE$CwO}z-D&P(^iNG?;km;i3r4_@R98c8W;X}Wy9zmJ>@+|Wf-`^+3-_CE4&H_WhY zVn`zFJ?*RB*a3NRJ`9Fq8d5G)r)~l($9Gd#z`mDxX%?3YveZwo2)&skwAQ{l>b~u_ z$5+D+>|w{p$A5isbo{&lYW6AUowKLdfblX0XC_@-F#p_YWTBc^OG;?~NYmK~*6{?t9xwlne z2KHHDMIt?Wlp&Rs!}I0seh^0cS>W$q!~Z1sT?)`h+CBgx>fJzdr6#p8(;jv00+G9@ zZj*vHu4tlqL^+5+2L!kXK_)@ITVf`&I7r}gp-K=d&;WZVtXbrjH*>>l>E|N*kw;B% z(?H@{0F0C1dKb51HxyCKniGgd>as*_C#-jUc?Zp4S0)wLzZmhN%k{3o%m@%BHxBa@ zrQsf#TIIBdKTqmv7OKYJI&j%8f&tq_xDVT9FfVq|b+rg*O)mom#<(vXemOe$s#kyx z2h|7`;WY?(gTO-Av%XyPI-i{OaM=F*^+dpuTSOoNz!nq~V~hn)jQ&08cy;53@>+gC zc@0aTw1z8)qlPh1T*DhEtzi$8+8k2FIVaX~u4eGtp}b2U&Amd!b)0_tvUAxUy$1{Y z;O<~LowoGa(YXDVtDzRnK%=&_5ux^#T}y38@46Rbad@2k@&8 z=iosoWQPYfwqfUT@M~UuRH?rDFdlGws6MV#x6jWz2BlD~*S~9z-Z4PlxrM%;L*%_5 z6n4v5w7h3S3ucHcy96vO`au?p%M&GdqNXv}2N7UR$$V4R@}4-&Qq-g&ns7?j5K^4W z9ijFsbki8|lP!yVz-{oqc&f?)ebb3=ZDCW;aX~o8R<{w@HXcx~tCnV+aQR8PBkC%W z*lTT7G(3&`7Xhty-{kTP_Mbr|NFi>c=Mm{vWFZX>Mhhu0`a*1NH5yoI6vE1mCD;xjxA-QSghu@Qv$@i6 z=uezwP;=!LqHc<7P32ZS`ee=!=F zUuht@fGcI$#JyUhH!my)CR^pY&G${MZFMW%i~`#tqp5MTX=ubB1(ueqq3DD;8#fU_>i&zI`*@#+s(8t`sLM z9CqHqa;O_B3ecv}@Gu-y$T-Pqt>*LOj;J3f=oxVIsSMgBsUz>Cm!6zTvW@?=7?6DW3Qv5(n|{sC4w)z!AiTE z`hJR1Qx*f24KyMsEJ4;$H$P{!-}cYwTu|lQ?x-{m%wm{Y+ffhEB-dbQ5eTpgS#VUq}%qdnbm<6H6IPt$*jixCpf1}E)3=H#P9R~tqduV{=jC6NJ6LhIVhQ#|`5Efb*MPkge*qDT zB@~h6wN3c+PH`v>+9}|21Ze=?_U};oS{#3=K1`32p`VNdqU zafibgLlAQ0H;D=-9KcqD1!To%7;yEYn`~JG-}+I?-o%O`u`3)Jy5&(G(4+7GP@RCy z6fBl=G{mFqde+(zlT7vU%;(TT-ZYkEEpi~6K29?4MyBNBdoBlZ6QKNtW2_vAEx?3a zEYo{Xu+d@>g{?g?ih%fM&>#bXCXxa#X;coMVghOI^jBJ+45tSQq#+5cow z;c*F_Q1l1R_#K{yoP(cuHn{xI9(PZ>z3%uk7`$i0!KhfH7$kx2*^6lw(zHhfEgLM1 zD=<)20VCnBLQ-)B*jRGaMyg^>Gg^|BHdv}DH`uLwhHtj`3jsHOEZVhdV99uBF5Psd zH{#N=<&I#G62mGEQbk@fl&h()w91Z9!M>IPv55dw1h3%&h3L6)Nqqi&J^fj@kTLM9 zXi^lcZ=>cBC2$!fP4-Ha$=p&JqNvsACCC>4LQrjveBbap(b9^RcrwwVI>yO$1_MCQ z;im%CXN4AtzLS7W2jsN!Dh(r6nG*zX6uws37ZUKRJiiO8axcKBl=gp9{;rbm(-iF zeeFpeHcuU34UFoVN#=3q@Fw2K52k_i>#^pQ--EN1aCwLOg(>7a0`_QB{f4wFNOOUb z#EU}Y;bY#kJhYZ!wh+d}^BSMAbmt6%v>4!Pu$L>&p#o4WL6DpS zM&tlbg#fA3>Hpdt4*Hj!{#ZO!g-;ErkAvZR>aPl0RFy6CQ>P04z3U*=Ku@a}f!59$ zwS-@T;pY;XH{?nKF23mYG#cIx+ZW?1C>NWcIUDpZx^Jt{Twa0e z4bIxVbD9P1qLDF=-4u39u;OH*^Y3C%0S>}x zv|k5ZK(f)}d;pp6eiBc=An@0xg;`CfBZ)I{W9h^ac8dUy4IOWKeX<#(i56VT1`1#u z_;@UkI(`saFcibz#=2D$-;kwJ*O9_ez}Q&Gl~%Z9y16f?SIe|G5?u=jdqMIDFl7qC z2DBFBI(DaN9SENjmNe^Qp?z!PC?CTSvep2YHppu_bKB4hS>Z_qz21QAN$qH*-5Ym? zeOgmB%fGC(6hXH$DuqtRPoqPsFdf7s6(%HrjmLn_A**A^mXBf(L#hejsm}R?v$Or^ zRvY?Y%4t-!Bf^eo>wVn(D|9%excB~9l*Fi883)t*&A9Y3+VpEtU?k^z(P+vHx0h4r zzpuKTu{gNHWDip(g~;oPDT?5JVj>aFffc=EV@>OT{4Ma$ns*c$zH9yE@MUZJ`TkLH zyfu{Us-Yh}$)=bVLYWT^<<^Y@3P?P=dzL%z=&PfHKa1v^z=d3Qfx>5D{495J zPWXU}QabRUOce_+i&gd!^$a5+9N=(2qk?BU%4^-GfaHcH6sudE~kouNLBT-K+qNgG??$dP}vEWQKZqVwQwlFJE*yf zdFpUl6>Y!(46jAg<~9_z6!ahX^AE9_p(f^oP0=kPP;?rXC47p;K>*xN;{~S@mJuWr zFwJ$YiyoOO-gcSmT5MXWh(q_2jp(t3h4M_ubvBxt-txmFTdwl@9?fP-7gRN!F`l}x zPSx4i6qU1ei5hp|!l1@bzKCf~m=63VMsDl^uokQ1{M_LbssY%}13ZEkhYj>Z2An7A z`^I9jz=s{MNY6UrsB8g7)!g+Y<<0Dusav&3M(?{HobwC#c=lc#mpGD}U@6Woa1_&d zfXv1er=%Xq->*inRe$)<>7R>NM0$Ivl1JzFpuJ!R-Sb|*CNZY%v}LUZJC0G$(RJre zls?B{j*TK>cBE{jFw@64 zD@QeT-M9V05S+N-a4;M-j_vsQJN4w^0kV8Etti{u+dJvxmWEV~Q)j zbdKE28`{RSly&{m;dU++Bt_<^@cbK<0vRuLRJrxyJ8AJ*18jrDH@b`v2MO#K@>Yg; zSXSf@VXAJNNh_b_vOPX~m(L{uHX302FiuR!<=jM04#v#kG{Ja2DhbRc1ND$%nvsZI zFPPu&dSHHpg@sYq)v$f`+3AkZ5+8qV92^~16%*$ZmfAl zA>dJMJav80uPC>Vso$&>JrZ!=A`e4$<^Iji=T0?vgg+iK0l4>NpvRZ$`ZsK3!fO*y zix>CoM=q%cM`*bSpfKh%scK=PUA0zqTgY`Qe%QGfc19q>9k5A1!$KiP0XHn5I*O&k zCu^G6=Pj~L6e0MJ}d*+i$t^vWt``2f~SZlmHF4)Yy3iSvdi|T zuXNI@4GY~Xr|820quj}slqgi~f6W%qdI%xiFY*5@LOcQm54C7kgU3=$1qsr!%8jl! zb+n;jO-RR`(}kVfiEu!Gmsx69=~hqkD3K41Nbe<%=JSaQwU2AVx!H4>Vi<915VB#1 zR~v38yba=ru1s)tKINODwHsV#F?(zai3g@Zval9OUHXtg)o)pps*PisfeC4fj*;%9 zt5H@#=yaGCQ<7w0T4Z67N}*L^a74yALW(&W_4DW&&x)}^iM(0Aa^ui(7ZVtDlqJ>n zbU(`1; z*d{0HBr9>JlHgJIjoKF-=WN)!a4y=tQGR}6FRPlxO(i%qU4R9-x3||4j#HdnIZm_5Ocu>$40ABuft4hV zFql_8K>APP#6hrWpoO#RccW@M)JuQ5T^XTzRBL_6&=hu=;I~XV?qm|n+w~YM!=er@ z%+O{pF5^`5S%btWU^dXI9eh8*D-MfglPM?~!wej($(hA0Fl_Ol!(XjB(t9>K#>zE) zk{v^xF7IocA%`w!tn1OqZ;7az97O|F_h6s%*zOaHbCjUxn)rPOG;WS*i|?eR&f+G) z{1jP7pqHhL_G2UIEwMB}%#k9^a}f$NWr0FbVK#Ct{1Gn*g1B)WI8Y(LL^T%uqFldQHg_m9=8SD{L=c9^{o;lcMsN_Q!mV*}A0+XhYZCy0JOM#Do?%bbZXe zbd()fX{(|PWGb@NLFw#s>H471i_=Y*r{?!IaHyZU-uR2e7X)ZeHKdlqbuiDUpK7&^ zJ4vE>rY!U-er?&~_b&%u@h1MW{b#sfbZCk+K7j0`+5-IdSi%az%*#hNTHgW)FlTSV zrxv{bx;`IEg>+%v<=vG$DPTq<_?qDk$6nMoI&{zgP3MPp8fDs27^YU6X;FeXULv+# zy=^|zDz?c@3c3>OQb8%J2h>(A+tZp=y5R07+0;f*IqkN3soOrwuvOX3zu5sxo>T1y z)DfsMztWS{O_g7jE=;_j+`K&O2A((g?3*S-CF#?}U^`YyHZGA@m~WdWR+7^ zZ*+=+dl4QaX!ENYES6ZdV;MMXZ`y$rbjWl2>0wJj1-aN;uL#^@M7pMbGTk0*Ro6&A zU)|i7ie76FGkFK9p07!KIePK3Vy2mIED*G1VR*W_H1K&}$-vmhi@zcGMEF$8K1Epo zY@gJN`Ou0U%e?INyO&p&PW$w9*!fkIo=l!js49LkIEkr0nl-7okj9T##d%W)vm5wE z(PNNkh1=>%RUOhh zNDlMP;3zsK)#B|K1McfMI&~1)(v1_~D*}G`x)yRO@rTwh}{ z*)^Kt0j=JuT$p3RV696zN7bZVM_rdrLp^{?Vqu7?rb;W19g>jIy;ay<@Vd6s9}F+s zJ>@}B$*|LFkGsEi^6#T}gW)*;l1CBgOLsJQ_444bXn1jU^y=t{LWhQ$m_;jxPK8W9 zp{>Is!cV_Pp;VNYR4Uz1b{Z{VzJ;zK_c`x~Ep)VbXn|x?Wwz4PQ+OV)J~r3FnUuN9 zWgOiFmCjIDO#(}*^3bE9v8c#Ms66lV#}sUK27O>YJvSh<91{kh$O6Cz1%c-c z<+2{vsWkLW-7K9&F?$p>p`{X#&#%z)+YCEhfPdnHZBZkfdWl@P!Q3n&F6Dd1dhC@% z)iubqqfauC@#GgTAX+XPy1we%Gym%FqY%ZMld1li7CWCKtIuHs)X0_kW~4d2J~VE< z7Ir!i@jwQYne31gzMUt~v6&rJvkbKJt-`A(r?uFC>k@fQB~zX?=Gpcx7?pURv^+9i z4`aOXLp|hQuzLN2KX}tPbs~SFPM3A942CLdc}GE5$=b!8U{Ht!T}jm~DKxnz6J*8n zDU~)ly)++>rK9PP#YFF>9Q^asVpnK)w6~WIjK_SY)26tG2irb6k62LLh>_+^WaAEMVfup`95OVzr%lf_>msCV;L)=_nQQ*WS}Jo#S>lD zv_AC4l|CcO6aDEgkU4GU@LRn+VLCzGt79C@mn*$e?U!U^+R~&cDawrDk6{!}LH?j} z<;Fgj(cwMJ8tSM9?N|8X241Ga_k^xS`mH{_c<|mT7CFvI7Lw0F6%5??4D<#mT<-MIa>sKWaUrUD>h>1+sH zkIB7}<4BdLEjM4!I2-i)$o)K3mH(BLTIrm#0Avv?8&4)lzUzk}th**>1${aCYWA1K z))o$puh*ua0!abxj|yjfuUKmZ;Os;|JL*g?V6dx4VAWs%2AJ|JyGq;-J$5VZr*m1| zDIyqhipwPI<+}2 zW$dpdp1LiT0!EL33xHzGdeNY2>GW^YlDYng1OiyG0B!WDYTt}BwR<3f8% zn?N(aD~~5_K7-y4VaX~S%1Wg8U@dcUBcnvgdoT$>xj8%F|E)Xpc$g#Gd=g{Y-X>3e|^d%unr_mp5%wa?NSQcO1Pe*AYWOP_*gjwgcr8cDXse|9Ojv6%lnClNU!P2?HJn7z+@V;>CJJh!?G~0l(>orm zTXa`?ejFnCI0jamISUZd;MR>d4!^>TwJ6E%#`Wc)H}WW=?5LDXJzVHYY^z)Pn*6`q zZ+%2MMlSr`q&f~6-qy|MrZFe?nD0|VsRnaBk0$QCZZupk;~2ycJiAKYMoMG*6F&Yy z(ZS{+JqWCyap`)8!ew8y2HFvb4NI<3EhwqX1T`gw{~UR@oCjV2dmqfuErvxlhZcdG zUW8GrwUdoG)tvpV!O~seftbnpQsVP5p~q=V*28fN_kMz}j%DD00Q3AnHtFS!jp9#l z4^MqI_|bPDE7889%TRX6$+vwoN+ipVH8cZE#a+~0V@g~vd=>4eK7S-P%5>?_u?bt8S$XBG4Y$Q}Jr zBo2r2fG~9V>III64(_Cf56r%L(Oe_4n2Vqe%-|DWpC%5_^3z(<4Afe<7+h3mDfcOz z!fvT6Tway2{NkiD0sZITpEjs-F%=9wI`aUG(kl;Pa<3l(QHJh1C=9<$WPJceBY2wD$w{`8dvBEkEH&7ul9w&n^Is2FghN&ZoIK;cIIQ3%XCW8JF zjcEm|h8SYE-9Zk)RFs_dtoW8#`>(hzh=1)%37>VJg73b=*UC1SQ-m+Y;6X>_xtoVc zt05qlPQLliNwzA*<^b80WF7{oc7SQS_#FTP%AlcF^eJg?3OlLv^S=v)m%eeE8JU;9 ziPh@QCXu(p;h?R2Vdi(Wlk(*}Gsw2#K^-&tM)#3}>C})yTKiDw&@CF{>LIrv*J>Ae zns7r_d$VAg>YT2);cJR31mBGM#EXGE(|fA@+0?n{}k^l^mu zV7fgF2ai6q&pHk5-Ud`snRD8?K<1Czrw#4q22|stWq91$9lygUHbWfi5NAWDJ$5>S zo+i2a9gUCpDjlar4X?nSXjUltRzpT%iWRIWxj^|P#(P@pfY#grEWp{>`p!mdF$1^3 zDx?cqq1i;+6O5#uRYK@_#j|MzD@(cyYZc0rLCoVhEH;W(dcqEul;dCnSb0Pm1(x}M zJKvpy=2>&Cb5f3=blh+W%#&SOf%%fmmsd?7dRxD9%TWo-oG4ShD9(EmEO{>t!u&k| zzO+c=GM9Sd)MB)jyLwrQPp$Gz8vk<7+`Z?mG74)H32&m6({sj#jNtF?Ja7|--!(Qf zv9N-C4tg~b4@uM?dqlSVA7&k5Lc1w=oSo89Yr-%P2k`fPibG#qk?O32L7&EOY~Uu0 z4TK4yH5U!kM3SrMx8J3$8y!B#p4z+S|GSi=ua%Lg;Bhfuyo=|k&xJPlQ%9p1daH4o ze%)s2eRh{kj>l};zU)@x9%v7f6p9m}hj{b{czsyTXKyf>1}K6UKDNC6kUO0hC4wjG zI7VCKURfbn1#4ScmsC0m8XgIq0c{92nndMM)ZYb9$*YpG=nQ{)WQTEWJYg;a6(a3I zjv1vlpcR~K1qM-G9-Gv2&jUNdT4TTuPq5zR4r2b5A#@uI4&h+6gm_)t@D6{m5O2)F z+VUzWggr@HkpN6>! z5Tp(oAZ`jbK{o`2Kuaf?i$rQ9mDC&d-*-n!vLwG`8|EO0b>uy~d+vp&9EnsWkg|)L z`TXiO`ZzkmCm~@Y7##^2=T!pirCR<5u{~bB?8vMJ=u3>%%PQ^55{Q-N*6keOhAOr! zRz)FflnZ0opQ9H&DGP*!Vmgrey@7=9b~^fNIzZ}xcth(yi1MgJ!h*_*93LuaL?$6& zLR$PyAVU|r@?6P`C1~xQ^GmKlTC-Ut4p?A4OuXWn1;{`O+FOa{f|l+$!yh54!I~}g za;zkRd+mCMU6+VBItkdr%juq(!y#7H5X&=d>aAJ zf?4smlj%XlNCR>RGKT*FBFZqV`FcN6IC?=J<~|h8-zxzY@kjK4X<-Jz9q&l$koj!% zJ3`h^J0t#pU4XlgCPP;1{OmqOLK@IJ)kC&RIdEF#khBKc!$r$=LC4BP#cfw`d3Sw1e4Z`3lABMn>#I+*u4pztzv^k-eVHvTZx-`& z#O{heefd1QxVpTWeF%pgMk*O`6ALlMr%eHKI-o}<%kEf-KFb}i~UN5>&E8bxB8QLP6P8e8} z<6to~ge8qRgU#5HRU%~8G06&>LqsS$VziAA6ef$xdf}hg*V4ST(&uu&vGvj+UNsb) zrVhQ&hkY*W@t~PvT(wjC3)lh*^@vq$SHM!COBjOymZhHzMn&JIa_mbUbp9JLgXVBC;kOu0fugjdrz^p1&t}+qx-0?=_coM6! z%m8{og}((PL=S0GI3BqcT)I6POqE;A3%^^KpZzBMa9rdV)wN z@4oO!14)4qG+%}6X$2C|u&17$vNvHf*x|TGp~Bcfj`Jwef*xq_0ClJlhcbe~ZRh~d zWcEqtf)z6AxEi@VyP0!!Z9+Mn@lH&q+N!nqDbDP@=D87XjaM~Ne=_w)9vW;1qkJz$bWInaGl-TP@AP3U z;3e}m9A364eCLUvVv|3UP}lD6E;|`v@#Az=xp!NVygsWh6ZYyA`(Yq^eOABLk48rz z6Kn%C$k+dbpo;z*9Ce_YxLJ6dZB$K9+%OQm@2?o`5d~;3RVCB|f1?8LId?4PEMECW9^Nz zF`dK?j!ri*@t_k(Fes1r9lCz=v;q0)PBG-*?fVaRXNORR+{9r>1i4j)@M*&NEK1eQOPthil?xb;&NWc+{Wf@U|Yo5qv zOWhx+iOoFmTI)cwWEk5t8UM4RaYfgdM#KOUU99E1k}mNdpDLc`?cjieQRqnB1fnc#+HMsPMrE8C;CN@!iFM{dhPzLE`)M#{XN7{voSbkuwC zA#}n^Y>VD4<|)_wT~7W^JDeKIPM49giz|7qi!8QM5cgije^|2n!MS4OEb*eEfzlL= zHg#`^IuSXN4|;aqfi0Fj{pw8?j?q(j$byvEt(_TTByGzbFz<26XRK8$Vq6iAip_CP zoaR!qWfsE?rG}BGeC4RT+yKfPh$7Brl>_L3Iji{P^ZDhHAA8icvBHoqi*;M)xgMfydfoU=v86$}-uFU9&FhhK@gc78|H{n&&*-GugE1>`<=U9< z3famI**1Gwj^O4(5N#V|$fEr9&2?sy8}cve3giO1EO?xSQNc>XKoC7=zha=58Y(5} zMaijzN`|xBKYXri^+0sex6UL??%pqIOqe}u&3X}yXT}ue3ztg_TqYnCFfi?)LNLbJq zuS-M2En=1O5(yA;b-#!Wxets@y?n@^_i+JPON&6Ri(b zvzTP4oW+O69cSGrCEe>)bREAX?0a`myRfLOm!Rsr^zA9VG}6^Wwg{p@MA^%I8pMpr zElOu#czoEsit#Ri2@Ds%Nd94RtekDiYSZ9ZkqMkgX|LSU{*D_T`Anc6&LKe5W1UZ@ z@ubd&)x~*w*X{m;uVtytvBG|~|9@PX;^Z_NQAaEG1J#f0k-isroSl!)3c@f9#?R|h z1a{HVH|R-p;7t(kqiZu4|D@7(B8up3Q{jR_#xoZr_i zN7^$wAWjhJjtV&FiPtcPY)ou=54O!Lq67<}^n*k1l& zcO;P`JF3MC#$kkax!D1DoV8X~&IK#OgFv^{M0(u+Y$ zl*K|KRg!9)zJB}8kd!RSO6o1Jxj1~xH?Qv->O_>X;3d3%b9?t{{Wd+pg^+w$r_VD` zE&JSYP@7LYw@@k#1+OgwftEYSo~Op@4o}?f+U;JFVG0@KnlsDSqoWJRRnvBs2m68{ zRH;=1Tk(ZU*mPydHQJK-T_p@?ZaXavFld#KmTRyI7l&TKR?LE_R96?U;m|T;c!9SG z%5)pUF%M~@pcYHWSzVhID2H2W9)-JDJ_C3K@Bg{I?cEdF5EkMq2_+MCr#W7|t9X^m8F0PyugJ9RwEEE2x`Nzq)04R4fwz-qHC>Y^O$;eSaY zF8pC+$K*41dl6QsA|Lulf8Lr@ZYNY8YqIQQZUr*GVGop<=P4#yThZ3obkT4F_rg|C z2r?$!c^KnaN>Jp%u!cv0hoT|pZ%TyUg+wW9QiN3UkFQ+g1ds&*5RT@C9Aj-lCk#m$ z!>XCy(!8jcoI;_g^s*jlnW_#SP$FHPbAv;HIe%mI6Eejeb{Na7$Whrf zJ?()c`wm2d-spTi)Bmu&=fK#;>g}9O*xuXxE^E?nY4Rw6;d*YO4rz=6NA@M6goPiB zfS`$w-*+a)@wZkmt>5*Gsv0`IFryQ+kq4A=6CX~;5)x83V${}N3uTGhHSFnj2dzV^Q@iYY$Mn_|2- zd_tkcIZXuoFcIPZLb<7c7CqeA8Hv?h|9%Y%Iyr@(=ew8ZOSrlML8k)FuHjD-F@E3) zY;w3px8U^D_v8vF1-#^=M11ry%@81d%yj#xxtbn!lm!Cl?^4Ha$0Sajt^jTv_(e zm^ij^TdQ>*JDtv%b{wC_6JsJV=UU8YIy^uww2<+1I5h0967pUPxiKc)YXY2g^fIn3 zd$OPCTo3B??LFi|59v;a(}6MR|6&sAeWzZ3+hmg!k2!-Kjv@h}uCGeij^eP?hNzVdoXF?zP$df&>Izbfhk0cMTfEEKHM4nkau@pIC>@(u8d}yAQ%G zOIAQ^$86!dOST30Qatbgvx*ydh>BNX>O-)F2@Nrr-c9|4J-B|#SrQiIzo}O))pD%q zr;B=`nm_o2Jtr71$K~ zOas5{kHEuN;iur|I_c)!gfT7r2MzpcIv9X5J@f7eexZSn`vY?{aV*q+Xvq~T_&c3+ zknD7=o4z@At_{0q+I#qqI=D3$ShpY=oorlC<|hrjYYi-$BKb)7_fNu|Tg&c^bMU`u z;Mc|&yx16h&NSJ>&vd)`&uQN*8(zV6y9&Nnvicsb+f{=M16ym_BV$m2>vq+=>rb59 z{^VLkY7Va3RoncP-h^eFfL;DrC+EoC|O$V7@pd;Y2uX|BRHF{pA3D{hB};zc!zf^50J{5Yrvl&nfeX z{qIPQu7F|xsn-gJtMC&ZeJB#c$m0==@DhC^=J)PnnHq~==|hDO`hK`Ajk(Syg2g$NqHb(%>Zl(e@xWdEaL7xDy`l!IC|CzFLEt3CFj> zOEQ}ffmWfY^%|U<$x6fw{(xd^wj)QuWiznRaj}8t9m!gRY2Z1syrmhhAY{pvvgObH zFb0nxf}{kIgEd@ST1PV@B8FaIxdb{C&rCi8u~d(;!#fO?YeA--=TMu>-1$8~WPMkn zJ6P|1sQ&?)@*rJCeJB> zQytGzFIR}A-xEq?XVpAh&oF%x1~&_XuyPFa*q<1XIAq?4WFpC*GdPa@XP!n#T;aF# z4tQb3B#_k;R6z>HESh3^(u%H@{RfkhzFiH>JIAnXnGK6qDT;V>p;BYdT&NrX#}_I`s>#Fl3B3yQpOr3nCXno3Lo7AK_M``$vPY9=Cv~&WD1Q>*MOd;Ry(g6ANT*2fiCL#JiHW% zj5A29Vh%nSEX1;K3O=||F#*#pMRtWXpB3}^US2Bxys%0e47{h3d zt9G=YV3Wd4BSlsXUKmt}Ioi;kIWfBvbvaPpN;TNEZoa}U7RPS--7m(Si9K|gEgQ1U z&*mL6mQ902HgqoD^EzT7XpNk#ZXnyg`W`@XT89C#GsXV*@4+h^WUT#r z{_K_A4?)|JyJcGe85}#Mm}RXJcs@}_)4`jAl#NEp2D>d)VCSG|;jrHYVX#|MFVWuQ zP;KIF<5Z~0exUeMGtEL!$W;^KMP0JuheaG|(8d#buK}E8ge9T??}3_=g$pSy&jlyN zo{_kKz#}n#8Fy|yh<$oN`L#fYx8d-wNaFoW1+}1GYlnfq%haN0q{gU-Ox4guL!T8N z@yS|6Er(ceN=1E&8?$a(1Rj4>vm$|ovwuSaPLem=fIbFu)Ht8qGjX5BOJ)vNF)Vi zhZ?+W;`6AkR_aESYzFpdOL7|$6fCL23zBFD=S;!&cy&Do0|*aIh2a8%c4{sN>D)X% z8CL!1Iznm8+$7=a4fs*I!K-S-QLB?bTR$u~nrz?rv3@*-s*Gbam93uE2%*B-VNRd`QIzZ${BN01j&Rqjj zwyvpM_-VY-rEkjKZ<~i4r6LZz5JbdB9}VP9s$=weX0KK|F;29@blHvLUHy2^9Kcb9 zfSw*OwBSvO80r!71Vn(X2TZ=+nCj}fS38;LLDSKfk##!)DR)(Xp=p1nd8Q|1sf0cy zL7h&YUaIW2iNXzsG6g^B(C*K6`0XY;P22J!6Z3R{P8@IeTr752J`3Cz*9VtJN2HL7 zHD7O(epmg$#Izm5!ZkGYYfzX=+xlXTYA2T`y6cXns!|1blp}MYzI(_@8S_4U_~30;qq*LV(P&G~4KY zb@~UVVV(1QoJ46Lt4Ru{hOVEAbA18by;5)#>$SZYI!mg=&~>6aUvV$j1yHnxlA8vr zEc?>plfv1dli3j!Wm&5%zN`>kp70ooVJMBg$~so<_EAa+!YWIs-txErF4CxOgnB|O z7;L@#?+&=P{|9}nf1tSk0eGB^S8Z?GHW2=7|B7pib+X|&PP^s9j16!Sr}bL9g`4bs z5NL_EiAbbMQcbd=|9y9)EK81*R-Yu^bI-l-?noW?yU>SomECKx*;)wBd+=g3`VoH3 zrBHAVuUX2ZLeO#dnOMu1ui;{Ld3pYBISpTTpW(NVyt>pq?!(nq7}# zA{7bK!rntFq)77=f*(iU^{RGuY-h{q5hus z)v1nrO7?`O1>Zggv&C$FJ>U5@`wRK-`N0#$3Z$}yTg(@>NwIFQ?(CNKmx_+~jMaH$ zU73aiGZ~NIbGK8LRx-TI{bKa*xZ82x{fyZyetc3QhJMT}D?cJ>!(t&)(pXQ> zU*|HS?1!jHH5`hVOFOO@N)%SS5&v;qbTx`~W5-Z`Ft#OMbKJ>@djv({@ONvShUvr$~aL6^o=J!zY-g)CYnZ>ZyDlNezc3XFEEKi5K z_2b&e3omDHr{Qmtx3i0Y_P7T@;9rh@A=^E8@-vKj^*Hk+4%eFVG6LGzLV&{&NEYH9WTDmknyjo7Ci`IOkdjJsWa zM|B5{1>eWkP*M3ER4cbP-l@6*r>o&7qe7$`%RLhPj zQ(2xc`&HKlq;lY&hg9Qo9sF&3OMEZf?8ogK?EHp=n$-Yv749Zj<4G zHGa_5~zZH`xrB`n|qt|2hk6c;MWQygN<3MG$;{>UGSLwjd?^`c&B zb;D)o+&!piKWXAd8Kwl=gw6s#MW~Q#UHyJCTZY#QcdT3l8Fh7r9bM7ubsIGN`MAed zeP~oTTfTcUU68^*-A(|(lf#+fXfWSyjx*>~eku&tYe~GRY2vXu1$K($C*r)aZjcqa&=-bRX#f3g$rxy9oPDor@c# z@I@5qe?+q?`Iz$@uVsFvp#i*B-@%Fl)y=r#UFvb$P^>ZUSK^S~$8BTO+uOr~R!Tq9 zIxTJasIY1De02k7tZ~*jos}1Izmb+#J&2!ZOe0$X#RT)hU(-@P} z@%L`u9|4KlfW?nR&1K1XoNzbm0WFy}U8V7?tZBLx$`s|O)+ra$v+GwdqJOK(LU+Ft zF<<4IZ#sW=hh2HVHj%hy@VMq(+#O|y%D(=p76$DFPS&5%e^wc6Q!H$-8sL=)uhpNw z!0i^71H3D>XgJ4F<6H3F*G4dxhc(JunCMg2hEb`osgwai$O_4&u}JL7c7blnwS<*sp_5Y z|K0Dddm@u5WNbW{@v~qLNg=^;fw6j(Se6RSL0Y|L|AO918nKX&G9fK1Kn{XllE<3K zxP-gYX>`Mfv)SMg=9Pt^6Cq2zdXu`baq>t>?R*cH z8lIXlgqbMdVU@Z1+8l^$?8J)~hUV+YO@I2E))0>q$4RD~-W$4+dw?*xdge~!dXa)D_>p?wG1~{-x ztCC?YpPH7h-EbL!$h2NgC)4Pym&!Op5IhhDx9kqR<*W+%;yqQFy;*(=a~!+}bB}HC zP5?)^-~ArW&-nT9=DtQYyj1-pNK#{z1Ku2``a5H`f(__|3V$DgSq43i@m0G0=wy?ABU`+hiLE2pOi>hoTklv zG@QW22}~b*%cwu^`|^5?9)w)%C>QTZ&#QRXUWd=QNb)vRb;2=63`MKqMQXLSFAkBE zAtk=6KBTv2)Z(G=W?pM7z;X>mZ1E!S4UwXL!YbZutDU+~>x9KRT0xWHqM^`dBh zTh;5a0lQk#F3L34?daYq$Pc8UR8LhD%sN?VgJ~BVu;LFU-awP|l`PivzeBaDAXAYA z{{d7T@(#Hac${s}(F%ev6b9h?J;ecUa)dsBmnIchR|VZ2#&*;}PSGVkew$E3>vI3j z|DCZ&S4L3<$MX$~N0DQGIpB#!F4Km#F{9biU|{OBf?8V~$)XtQ327#Yy|B$rc4o97 z8!-U?rg8SEhHJhEq3F>tx2$mdtN&%glr|Q;Frl|(t5$Q|KP}E6<#4^1CAdnfjj*JE zr(5FrmvHAjBuU~{G8U0hy7JumU%d7POSNv4yc~F(ZBV;T12GV+&tGBb8aSm%o2yU+ zibO+^E<%>E_s*-Bwef=n@$cB@zy(CJ^2}H}JJaJt8Ycd5f7~C$)7}1299Ul%B=}9T zm_k;GPmq)mUaGT@VUFyBb;?mpTp1bokiB(MS9xR|QsS8^u=tAU!f5cxly|0xGlAxB z86oYq8`gDa0KY76(7Ix?!W=;)gLFl;bELHmzC(C%5gH&Td~kDs7#o44Y}RFmoOj!i zeU3k=Lg`>wquu-;mJfBdO-0nF=MBI7R0gfCcB;7Xp;SWkU#sTYq>NQ)ooR^adnpBB zKa={TLfGB zDJa-N>C+K&iI;j!B#GMo`|jSSwic-%dgJW4J2P!1bs{9(Kfc6w&o`6!A!y^X(qbQ~ zvHbaVKA+8DcnH3Z=TkTjf_@l42%Kfvg7Fo|jpkk%4boVUj<4b!%T`43Uuyx%flmd9 z&$ZxQBn4<>4w}U~f`D_g>!)};eaolt2P^!>YS^-DLkbs!(`(QYTVcO^s`L_OlM22< z+**`zYN=U<|H3rB7smREGCE303~1wPW#f^(~Odb@g69Y z?WydIV3#VMLM9m2%V4kv(T|Q#OJ(sgY(rj#Vvo@PI=D+A?sBR&VDQ-%4A(712d!UC z>^BxuWsVwqSSX-xKYCMZ6mx&ZgSet5HIYISEG5*&Y?KVX0rZzm2er5bc$~c&Yj4{| z^0W0THZCqDT}77MIB4p)g(A_mP+3w%DRr(52%213gej8c!5crm?Wz zj}|6*^BQIa$+NL>W%cLt{>RqgL5my^lTzZXNSMXM2{)1F)0lV(xedbmfTSTwvM37U zl&pNZ^{#zN>^Qa`$cApNX`FO`#32$VoV%-8mbn-XtPjTA98D*drIVL0NexPVI3>Ne zZ{K!%W4-m2SgZ&~4i5<3MCk(~PyG^u^T~1mgkc?k^D&bA7J*MuBh*i5GhP=lL*E4* z08v7>DGd@Y48*#?^;RGadkFPS(>0Cny@a-roSpe8Q<}X364zNuY;x_TaS{?CY|&oZz+N_Ih(f3(Z3CCs}n8p{-EIsUJvEluia0&&YS?* zNR8#|7awQF^o*^wtn?v*HI@ZK9aWFIs^ZloxHR8u+cxes0U}80Wr3~z-Cd5yBc@)Me(lEGl4A4PGv^)@z|fP^bRqr|7dO}dsNmpU--lbA*^O=yrR zxDXepoer~^kXPVaoC@r059AC|y|mS%&hZ=a4e54{b$Il+Daeb$rTeoP1T7C11$-7$ z`raJkL}g}ajt4dTOKUk9T@|{tuBJ0%VwuzBd|;3_UFP05zIW{$fzn*kA6=2-&bMO{ zQ0kK7ZnsUQ5e7qV z=#R%zx?TYV?`(jg9&#w?FL3(ngB8%^B!0qeuMRsg&>R|M_3byOT;F=An-iDT;bx7jfK>De# zy|=xpb^?eLXWbxZ=r<5ZLg$t?2<3%NRmoW)7EgVXsbzjNEz=kqg9R44GG*#hhf@|F zluz49LgNb47;?e33<_upUX&hsK{GscqVj|tNA@93)$rYlBSiNV?}PAHOr}<@!N>Do zMiW-MGPdnf3@IdTVlK|EK$eMFP*LJY4L-g)fg%KsnK>!(Aymk{2iZN&W*)#K7koKh3E&NDb!A|bpT~3&Ef#8U9F>XQ7%JiaiRJW$)f5iian|Wx4+=ZSCO0{pO29^OMIBPp52auNZT#%Czvg_4Q+oaR! z=sR}{H3BjP(IQBH>8ee7ZE^%fJtqrL0#w3y2i?yk$E3Ws;{cNFKcrY937vw)G-btM zhIv6_VC_H*_1zk*2#1tFjnAdiCU>^Qf|HeiMV zRqhy0m<4Igu_^@(`z`}I_YXN|K%>DOa^PLBd-h;Li~TVp<-7Fu2dEmrD~Za&C$tdLjdSv;)SYqJj; zm+-%ELArte%mdk5D^KTqEVjX+h24ajB>+-zyG1Jh)Ro+}`8qy?Rdo^1Fbl9EXIjJ2 z91cU%`N%Nk^eTk&xorEzpnRrgK+SGn>nuqWL#a8u9^?g!&lQ?1c}YVwm=h?flf{|E zTNBfIKRsWXg_{JtyJCFhpjyZzTvPf2p%w5Nd}~8)-c!xXdpF+Ko#fo^O6=}-@w2DeW+Xatp?L4M%VNr?q##_Mq-+2MJyv!QU6BII*{veDlpo zYmXmXh|IdlN$am%QZ>D3WP_W(C)O7DWMzY< z+ZC6+| z^J-xBw1PNxSwSK=t}grI@f0FqIx?q|)*dW@daD7oHtBBr$2q+98JOS? zO(%?%ol-dZgNfN4D!G2Zu_6%Q`X0yH2dw=Z1Bvasc53hAKPSN)+PNbG(aghm2P01YRYh!V*RbC*VME1Lr6_bc)*Ox7x5o8_&6tz5gasx@FUV7 z>vgO#BxI8_7tHbfs<%qn(AcS6-WA2JKXoJ~4`7lzXXpB9Ho+CaHfhGJekHbc` zk49si@zXIW)_g(Ma!Oe%z~A>C*P~dVU{)2WypQxa4J1%3!*n`8@5m1_~MeYSsbsUWn3 zJA-y)e6qthlHtl#3(x0Cv_qt3BmBs3X}Af#C;v+JS-s{b0QQ56-fS09SlG!~$KSaz z1t<0f5;WmJcJE9)X!^1=#2*UL+z&9xwvq@eqypV@Z1dYzcBPIKobV2S_%k^_x@boK zyg!<-qG3SqJl8vtJCYbq4HHf!3v#~f&-;^wVc-v!X{mg9oF}1pIqr!cDIt5~gnb7Z zKz2=)L*5l^pSUNGxmw}}@k!nw_P9?3G0#V?h9uZxk-eGOG?;%!y8Neprhc@d*gC{q z0b(l9MTehyK}Lncpph8kzZ5gw$6iWnb`sF13?%K1~44s$^(xrr}+uIQX* zgY-;wo7+UW0K-=w+RI#xD;LgKdn*_7t-#E2gv>ChDs>q!r-y#k0lq1`0e%?U@dH_b zyibaY9BZOVujNmTS1Re8@-!*#oB+IIvuiuC@=tC`ttCnpK=hZ52e{n(^I+h|JUj%- z6)3rw2lNsOE-&_lQ$I)??8dNR1$9b7&_d(PD}mC+XIl((QCo;tE|Xuy zWy);EWt%GFf~FtAC4Z6z9>OH~I=l+j>YK+C@ljV@#!UB&WqMN(6>C`j;O*nuyIJ#` zfk~{iiNcI5w{Ianv@O`ma_FMfS1^LVP_aVjsJu5J84F5d%!pl?kw zp9gS0;fKKu$^4W2;E35xR3Op?aOU&)MP3>Ze+frfBrcO3$Az*5XHNJVVX98>O;efc zzylpE^Yx;Fz3KrCAr(30Qi!SUW1IG^N48wu>1S;4Gxzt!+Y(ncP~DCBq(8QPHmCDN zOa4$|{howzDo!nYPdkr@aDWka-yn>3fS;41ud2;Uxs}nC`&`DS;8?pE9ggM}ejA9N zft}B%%b8^wm(Yie2Bz|{#vq4KEi-1I`Bo)Q~zB3%sk|jGSvI{J* zkVMYMn>TO9xzUv=v4TIoK7M)HK3(0=gEn}4zPew-BkY~(YS11ObXJ>y4uXYH(;M5V ze|rQUTwjFVA*}DO-AB@%mzQIg9c_Afvl)T`8!HM`E%9Q(@ayhXf7*KIo=_ zLiHZ002wT*#t!$|fbB!qhoH?K8mgm*%DQ-NG{u(W&1-<5+hk+Q&s)AkxRRejHH~1& z7oMwhs*k}=kC10_lC5aJ6*pVrk@hu~bk-qJ!+7{*bK;!ubLdO|F-ggVAO~z3-FfnO zdwq9zcfE>MEND~ef}snw3Xz*GXiFee6KLpGtGy>g78OdlTqw{cS6}jhwfsNU?JtS% zm=4Dt@m4S;w2nn+Rq*3c@e?mlJK6ZbcUb636QJaG*H?D{&w;}?R%(W>)h04-qz9YG zHTs%S1}y+2`$f~2D9(%CIW#$?O{DV!So-7!?W3hj)sPyJaeP%pXbxaaA?czj>PV#V z_L3z6J|(-b{~r8;2CV&V0X>ymm1QP7r8?Gq1fTwS-RefUvs@#XpX`7y=r*7u-uqrT zMIuK)2g_y*e}&F*^h*?@Z!VVc0adxl=uf_qRk8CLqLTzU;U>`I@T2?WM zPdQA(giT4bOl64?SPmbUD$*24MPJ6)ssq8rmEOgGQk>koghza2Ku+V7mdq4sk%ma7 z2-{Fy=PVJCMelk}lDnIao&e*W{@=g*r-R}sL0=IA)l#7-O;ea9~%G*QkH!KG&?>Oi%Z zcVm?zH7!Q{Om5UsMEP)%PZBb*rF`n;d56Tz#p?Uh{Jm7I!DFkxk@k-cpWMFH;2(?s z$Ur%r#aL7`d{)p>Z)Y9^;iR8~M1 z&NS<4m6z)@EoRpxecEBpHcR^YKSz4f{^?-PluIqpqW5Zl1fA7b+9+}ObniY7hA#bV~)WHj`K2!I+uaA*5(`c$}3~ zO^=%}5Iu)qVJVk3;?UjW=ENqnsk94GN~PVC#gYN65*yhj&9=&a@7M%_366upVQN z@%pvHf~cqCZ5W}=F6@J>p5kwz3q^6#+wXkuyc|}Q39jL!`F$F=gW1@V$Xf zSv4|fOXv*D4wOO}E>p*C&Q7!fUg&aSoL31eyhf<>!`%>M{0LIJn9`|k6cxOBcL|;z zU;=LVA=Oz)|3YV?;iC*ch1}D&LYKf+VDCa{RZ$z?z|L#%_%#Z4uXG)L4!%Dnvrm(3 z6h8;aYBRf=WX(04-4C+0^L@AGP}f)m?GBA0K8(-a0vDf*?zs!JPliw07sDr$55~u(lR6WJ znpNQBR+Gs`j=6;Y^DaEFhkpIx25!=7~qmxgr=UwoEDE135yf;VfSnmA+fCeFZ zu}FBFjZ?vH(?AeCXTM^k2r0FkwrVLzL?CehrBdkykhocEdmK;W?3&%#27~x_X4kkP zL@iwG^=RIkH}9?I)R-C@Sik+aF4k%;XEHc_s`?M-~)}_jdh^A4uVJLb0`DD#@ag7xh1pl z5x#Fx1`jsCUK<9sa2Gto7O@9*y4r$mKto4Tl+y^zL7O_FTYWFeN8VAWjq|`7>BLfGVZx*-x&b{cp_gIncVoJ8sY+dAt z@x&Fe1~lvlJxC?qjO2ZkdZs+PP;u%jk`+bl9pHzW1>d|; z(MPywT`qnJ_OVf!c1{l?A%2tP42)wsYqy~Eh3yfMkgO0aOx7 zIxXMCIN&v$uO2GK9e$@Uq;&iF=0-A6YL2F+M*RY=V}#7QssVVMtycSQ+cpsXIsPjS z4N%xZ5_?#J4NIHCi#v#3HgY{?QP#cw%3Bn5{)f`XeFE(VaR9R$gew@rNcEu>|Ba1g6 ztSe?^${1@|fs5P)+;-G}2!29H(4lfLlNJ-m4Y)ainvVK#kA>AF&?zj+%;{rSNMlnh zKwGvgt(;{DnoYC?y;x?rK;xuiD`^s_((}vqaWTs?kV@r6qRn0Qc)}nPZ)(Bh4s5%=3OQTxNpvx+) zHSnIaATs#DZ6!~XLJYw9*710nVL3zTKT$YX=GFu&YP3sA6Dm~tmr47f(2mltVzPW@ z$qg0;D*KexL~vCFyL81-y^NGGy{*2@0vEF*2#O&d%4-MGJFPKQZYY@l71@+o_Jz1y ztudE9(p2Fkj+TO=ol;%s!Lq`kR z{7n?Q6KU{-#EcgIt2gG7Jp*7nh-bK$Ng}*ry$KuV=E&Ew8S#33o?1Kh*5PumlbVq> zmHo;099?A?Mm3*s{@;NPJWQ0= zUsXLC(7OdC+403{FdTvZ-k2S&t)2IZDLOT$<+LIszqb+J@Qz^BUMcsMr=8G`d?^SQ zQO@r#(ISl8xg~gyM(fX;Ki`pd8~Sh&1X z|6@r#=I7t70(sq%Y{S@Ylkwgf+3u{*bm)VT9%G)M1}Rk&IirgG16;sA$_xd5*O?;K zx`8Yie}TC5FClWtKE4-toOO=R3W7io#_vAGf)|zEpj#1C1ReCgY&vUGIUBR1;oE1X zgp#+JkMGYfL`wx&bvmS(K5ve@ZIs-`VQAxc18*rB!+2g-}_;&YeEuW@PbrM8TuuabS?9-p?E~^YD1s`ZSV#cIET`(6L_3ukHHRt zFbsyz?kSpZQ37)C=$YsNFTTJsMkh1ebCa zAJDdx&Lb#I2nPKN&rwz}mT6@>RM0g1Q9o}rzq2g%+Tt%_+6Exvi}PqqJ1T{q2gp4R zBLyBo!48m0Iy(fAz3`B~Md(*a1`TfL=eke8@BH<)jq`f%wsQ={)+Zxj8 zllEa0#Mk80HGYn^(~5lo_L5&*xg>a;g^|H(!!Qhn&(Twu=F)|*^C0K49SVc7yK!xK z(MaTAJMCcX-KR7yG;POxF_!;N`X!a**`Zh4*Ju6yw%O%DSIz-B-u{YG|7LCK0?p9b@W7A^|MNz2fEKH|y z_lZ_o{No?Lax~{sm#S$<6(c(Rw3E$Qu{DTOD3N6^@C$(j$aA#~c${s`!3x4K3ib2I|T)A`9t#k9R>Cn^f+9u<8ZTQ)k!4AuZ<$E zx*&L)fMSAtSZmo_)fwt+Bdt{%+4kBkEVG-TXcH(RL&E-@O~Jfan5KgF zU^(;>+9ub6@!#8Oh#+uVzxVF-U5vG^pysE%*yp0icbDRPyyoJ-VvkY7jKWYmu7K_! z2xLnZJ4mdfRyr-nQ3p5Ca`^0zaUve`J%p!cQT0M;3@ z3xB2rwr$*MJFM;q!1lGn!*_$S3K1jtaZhN7Hag&iGZs!5g$#O)_><6m3yPYI5(<=| zSJD}*f;R?TVa?w7tBw=Ag8_J)rB+Q(+dvRK<6kinm#~VG0*6ZE1Qk#!Br2Li5kl6+ zUdLPP-Dr1Bl2-ip&a6LT8wVZ|2RLoz0<4R4fv>yZCnbWgdPF4)L2*VrK}f zGezV;FMf&0g_*Pt{saddD=_IqKI>UaU#7&y&4SliAIXTs5>lb8Tq+SmqzmQxAi&V? z@#^;SXLx>pfBx_^gDHbt8v$%OfXa>G+YYQ$4wRYoEIi9}^rm7T9SY@)$JoR2Akqq% zMJYE(kLQAH4SKF)GQ0vS!)!^uZtVQw`(=1>Gyeeqr;YXM_WZiCo_r31DGPwXJQa0n zw_OR47OX?&!~jlVA)TEB)P`=EZSpH_EI!WQxZQwaz~(cY?aFJvOESK+__C?8{nEF= zru_Kvickn$iDZ;ynxQgo3Mv113!HuF|o`F``7TB9^vrOyfVpT(w<#wCHs5`B~ zsVrcT5!{In9xO7N| zCDwa#2vy|uzVL^p)TZ&xG%~XCccWJcS?b6tmhTC;mM(>rMn5N!$`-^X64pX4aP4i& z&4noOpn}yj7V25)wb~c${^Pu?oUK5JcK zfdP1&byeGL+c*$?j=o~xyx4-1*p{0-7=7qw(>7>P7jCv#V0R(VB4x9cNQ0yj8*Tr+ zGo-9boCSg;9?qOObLQf6nJX@G`26PfL;5~C$N!}g$B)suR9R7R0Rz4MCNg)q9YoPp z96=19S@vCU2zJ;C#)U!qRUhNa{ku1Jd0Z$Z-xfS<8 zg!9OXN`Nd1v0();d|_*i$Og-oYJ+}6wh))-9LQrb{2_bBtbn!1ScQMdpsI3>IkA?| zVf0kMyq<(X@flM}I}rO!;2=!L^JF@{zMhOHlL-+QSnJ_9AAznSN9Ldq%sPTdTFE$b z%#)_<8e^&OW4dR#=I~Db;T7|BH<}_0Mc3+697jc*4w!jv1$9G~(jf?%lNk!?EC~X) zNXirk@7i>rPH018O176}RUY;G>g&U@`i;Q`VAHIIOqV-mM6Ejw$-FK%Fd_^k_N^He zZt1uE^OM0~ptQOuHOios%!1c|FoZGMfDE-;k!i(iNR-`hYOyU*&EpJq_)vJ`cnM)7 zLa)|H+|WFQfSu7hZBe$o^1}G^6ds108ZP<1-KiF3gY(zSKl#-Cx;(uW_kaG~yBK+X zGoR3nfarE>qonXvs$tPC82>al5?T39#XzU{Vs}dK@2MeGO!ehdJChV zS!{nGh!yf+RpHv@|6flWBsBOjTD*J>LkBxIv3rdO;)RRUQ*ZI?p&`|to?EODhki%5 zquEjjiAev-Hwy>j$>6yAto7oa(xJiR64blyNNf+ z3deT6>p+C~?>DpSm$hT=Xrw?g-g)-fd3kp7N-UFzFX3hsO#N_jGrO9Mr{U$`6&gjt z_v{0n6dTA`E_s+{Dpb3S!{gxKWC$|j3&CO_lZ4^P;J~tlr z$?c7&ArMidiG+Kh){vyY@>Q|niE6=}U0Kd&lgYI|8he1=;nnzK0QYGTN3dpF4s*^E zVE9ido#%v`ihIOpNPriQ>o{y%N^oGCSt!F=-&L%hEXcmqU*=&GI-t}UeQ}k_t zZ%hU3fsZHJY!F-av2X0Zx9pc!=jZ;I?0*;rQsu=$mGL_rP&H72i9}hv7ISBT(N)k| z4MPu%d9h|W3^DAUb}SOfm5bmf!vPH82Z8>0|DHYrtkXD>pw?Vh9;8eC*&uqbOAFW( z2&pTACD0Tl#0>2sp*6X|VtLMXuko|qMy}Ibg;Y27W&};-*n&z*rBFyjl~`qZadi;# z@!;s&n_Xx}qz zkk{~u^9+Qla~Nz$y3g=R0<^6gnG^6tQ8OwPBLS7Ws1t8ctm{e*GyR% zhA8FowekB*c~1BiGlQqoL2~%Rf@cbrX>MiSEgftHqD)G2pfYH*n+~JiZk4A+hJ;we z^P0A9teW?R$vX;9fB%(xc3bd0NVcPm%VX8Cts|SDs5(h%4p9o8bh~J}aob=`ZWO!2 z!&!>zX|PI{p#>s>4yYE|x#G<%X+rm+BZ( z%8EXtP!Nkd4ZhPoNh?pe z!%Ts52 zdHNtCRBYh1FniZG2;aa+nn&1*4!TRrE0sJ4EO`m zNZOOB=1CS@xM)wKq-k4k(04iMSMNYuLjJFt*`oZSuS5j;`rVAbS}ke(Eg6p|fj_$q z#-R0AuMX)tst?eEIjlhcgUT-)$PDAh^a7b~XzW&8UqDS7_0wnK9J2X5=i3+l5YS=3 zE&GvLb^R-Ftqusk_`%s^>N{JOyU(^|KIMArjbF#@InR5B>>*+i@SOgpi_qcbpfA#R@GzmJZP9>@4uVlsNr8jjelV z@?yaev{sTsnlAoKK0qW^Lh0WSk&yV~Unu4 zOYwm3nA6D7D(o!B(>=5^OAB>kc@l}`;9m_?68*X!c$}S)F$=;l5QS&+D-OD75Gt;j zLy0}CtIJB4g9O?w( z2waZAU@d#P1mbqoYS^A zpWjxzoDU3g48KKXCj1#C66E>~A(ijn2Qn3Rd$y>`vN(90l~X-y!!Qt?qhE30#W9A^ zHJ-Yq(4o+D7Q&WmQAHBcskD^-_nz!Db?lfl%Yp9Wy?5{IOtq$wEj-=tcl`0;dCzw& zql7Z@EXM-+SR*)r9oH%#S11qg#a4AEJQOKkvDLd(4HOZTuGn`q8)DRd8K_7x5JEIT zy&BnoG6=?(Ahfpiv_K$DbUcBC?Oo5sdT!^b%unEXZB0NJG@|^#5{ArF1#Pgdju0nh zwU)x?^VSs0UPtSt)(z`jY|#>yhNA;jo2b!xF$BStmlT zJ6{qwu#Oi>#r53fqobp@3DTc?k&5L4$$ucVwQteg0(s3Iy6@DWt15(Z(locI=r5dz zM&HO{b2`wkEPmlqH{6J2BviFPCX9ZowB!$-EvH zW;HbBG`2+t^4;A(sfKZrG`J-s0XNzRJ7MC(nhcV%Cwr%Z%VLk@}0p`4J)AtjsYGy=&IUQ7Kpx-{-JY_9rOgTXVY zA3;5H?zkU#oQ0504uUWcM%VTfldw_34Oko9Y2*M6p;Ksr)21_+czYWRNYucr`t!Zl znP$$CK?~R0<#an%l|HL1?H%FrNU_AB5eU(@5t&P+9^M%+jzeERwRF)BZ5IaFe-vi>Ik_w>2bkN4Qx2H!PMHe zsZhIA@UHnPU~G^?OHy^dHw$X?ruWAmWbt8T{CWPrd41=0g(+a!e99(?%8-n+>IDQ` zwmZ3t0eGC9RoiYGHxPZczG4z1uq7x?``Q$JOOOCYiv-DAfCaT2Y6)}6HOaNK{ra5Y zuHuy}q;UmH=4CEt&K#1L)_Bc^K7IY?%e(5s;t~_<`Nw>5X?^W_&Gcu=8k;xGU&m`U z$kO7?iaxOGC2s_h)F$r>6$Pb!K{5}y)0a_U`{^3#wqx7SNj38_nAaoe}-^B+^Z z)I}gR*LADc^wNZ|QITFH{JmULx~~d?Yk2*(Jf8o)Tuvc&p?8`#OsW%pqg9!2h(4N; z+5-fXxeWy(+M9okkH3ga~3CxQyi=h1F$eyy!BuXAMnRnTweapD+A zkUVj3LdtV@n2z2O;BVVno9OxeoMB< za%Vw&Dev97VJjY44?W?%AUE*cWS_xhkVoKXPweF?l(QG~-fcTsf-2O6BsT8WdO`x*^GPs=s1 z1{ZmvJ{1Xly}gwaEHqi)VN(a#`;f_na7WJG@p0H$Pb9#ULXX5w^-U$;SDGi;mW3I6 zU>c~$`s*s?!ST;281KpxTG;FEk=2Qw2dW}233+bEOWi5I?bQ}~f0S;aRi5W(T?$9eLvDiq-|o47Uh2gqPJuOx?~>c| zD8C1IoKwzB%S%a3QwVZ(boOv{bn6+GD zT*|3=DVb?p0D9*t?!Om!oSje03c@fDJU3sl(2IutfhP|>z=Gh#V+hTrUD|FWiPq1z ztrn#R1urw~ursrpl4XMyPMd9h+^O?AKd8M(dx;ols=#|h&v0{OfS984Xd73mPB--; zM1L(Jd`ODeIw@hG;=5_Q z)9Mj6vrB_=|<2V$4 zroZBdc^Ig=P}oPFP@7p01$N8Kj#jHxWRjEC!nG^g;WDH7?|Y7&HfcgT>aK(!cFyHH zx9@}xvd}peFik!^q*uKU^ei>5jy>mXnR&2Sq{?JZ@V$4C&4qWQvP?c_{j4nfPPJauz%_V@YlA9^YEN+GXKg|Qi?FIIZ8`QxJ31ejhw zJp6S_4=_Ay-T!_$x=F?`Jn!{R`#tExNGRpN&k@`bGg(MNP?S3J(r9)9D6%<+2?1-V zy;LAMa9>*OKpXuHtr@T_q4V`UTYMI(M0#=B^KVP!WHNF#81hl*qs(aMVVyY6&;a7) zW)i8PWR&b3Ryhx?s5kX1kDR+-=n;{<4JGwwB)fq57tvvvjbuCPU|d-pBH13I!QuuWZ$kSs6@Z)J&M4z`a&)fHf+S=L{ygZUG_( zQ7vF+D>ZxqAz&t-Q3n}Dw5fd(x~s}vNDH~Dk^T4nsw)z#b|U6mE3R|u&}izm>p!*4TEKM)2V zIkt8xwxnUjJ1v;hEe!C8d*tI)aykA8Jkp`M$0uBaCe7U4j}!WPFnlHc*@=z}$F8QT zmP?vs+!NMnkdbjID*Tz8k@& zQ#P2H#e(>e4RXp@C^}u&wB;bk|J=hAsRv!0-0;?mCz^}L(h~=J;7#CBZ*)Xx-BPdm zV7L9RyhTY}2)y~mt)_vh-b=V?z{M%Xsc&q{BKqG4ry^KJ7?)?;|I zNRh~vqIpKk@o=1|y_M*;txXb1Nrbm&dnO{n)~)WQw}m`_YfG$mOnXgM{=4by0)%>5 z)q8s1?~YLV|MBUuW!sb4ZA*1UWuvQPbdxfneY&6CUVcvRZ(z9nZ8DwSPvPj7v*R5W z?vhFJXEGtN^BrPeuP5Wt<#bG9zaDRp#T+Y6r&t!PruM?l(60u^B5ykgrKw16*jy)_ zNn!EM{vw`jkZ{#e_|mzfbT)?8qBH6_(udTbF>Le0nedz|Jy|N%IvJ7f10L=a>JS|ni~7mM_F^EqT<}*BwO~p=Xf1q)+lSw%Y+T&| zjIWdR^B-TXlh4%&e#AM6E3AV$ntUI3@8IC4q}bhk)ZVPw_#A!*yV3MJG&Jg57C?Lx z|9jzB{{=qbT#d8C0eGB^R!wi)I1oMCzhZzau(sf)$sX73p(v8=0!;(By$g&qw1`Ng zKvH&%MgRNGkd!US_HJ@WqWOF?^El*(W)JJyKKn|IQX z*FI>#XsEEW%6V15g4!;++Ao7c2qo&hIJh&vVDQJlV4Upz5GxB4c=xK>JSEAywiF(d zrcu@!J8hW70@V&%QM|6lny#&lD5K(7y)+gI8gQZsS63erp6FJ2-{!3~KM(8qul1Bx zS^S%C(J!X_T9UOtw8%VVz$n5Pg1yv>i-A_KGpZ!&ShXYn&=O`4vGsCTUgQ@Yn@&5m)mSo+0^v`%Uk`+>@E)z6iW@<5RC*ibeJ`D53djFT z=VL?hX9y6nOzBNbIFWw<4zj>w)-8@nx#>0Pcn!8Z>dUsPFn(DRtJ#gHN6dS)NObcR0&1@;0_OsusilaPh$w-IX-MgA z=gh}2WJzA3A&V<`EIL!~uqTK+G~A*yKtlm1KH=JsH6xRM>x_UDA5~qBJ2j zOc2&&Tq>wY`PPt%E*!Gt&!FmMDN&!42u-WsaflbB6@(QfU4_hhRo5g0hyD$bBD@d` zge3i5*s@30DCejbxp`%Xo$IK<2auw6v?gCV5BrK*yhjjJmMBYJZ7){nqPqQ!*UoK& zc8eTId5-W*abosp#WHPZZhkl&I#~@>6weTkm9CM*YC0wtmq|F)+~w7kjoH&=J|k9* z2^Yu)@$T%)ovuDcaLV#F2;(a+tv9+@xzcUmW`E&W6s0@zh+zKifR0L-CQUJ_4E>_F zh{C#br`Gg0p3-IRE#2aT|JuEFr6+@0(^M|L3McOgdFic~&?JY8FwUNGfg)!v31Zat zPn&3IN1y58_Uo*OEdIUv2H_CNgx>=(8W~D;Y~VOjdUzzCXdm|6X`S~DkxqKdgvbJT z_ zvNBaxCSJigwL?r&dLgayj+{@~4KC~`xFCLlL&~gKQX#C={`hpq`^hNz zOP-rVc$t!L2#u#F*RLHCC)-CRzO+cfFC;x7ym-iEWh`|%s6&Ml>DawmXH@^#&`Oau z4Cz58W(kW>+AnoYo5&l48VT(AStr0JE>z!)j&u&EekcP(?NvC|PUf&o0Ftz1GH3|v zV^}SPPTR+{EV=Qyak5U169tx6Ls#32%)?Z9fQ~?BS>%$z=45#qINcs+#t#9z4?8XU z4I_aACp}t*8Pi(kED&6`S4o*;?5>8dw@+*^yBc=i0Xv(l!Ft7H4<`z}3fXGo$(2Q6 zEGpfC(S?dsfQOW;a6=cKcVX?4TuF^-2l7tV9)Laq80!K+y{M(obEOB>4f3h)pfqT% zB=%@WQr&l?=1ncNQ;*yJUsS11XAo&>3 zJe<^}hC0@^RhZasduH0+*a{QzE0}Nm4hRnUhx6(EJf4RzAZPltn{d2}P2t;n)`OQ&LDi#?TYN)XrvDO7OJ?mtFGEM37XX377zE&zQAV$6@J;DedV%20D*qlpe zUt`bViX6I0UOE;&NdF|IW;a{zSv6F~`L7d25G{G` zg#{^@O;Rr!>?J|lP$$!qz#%6}Ps%S)2%|1-hIcdyW7$^7s*UxM1w{DMq5d@Bb=!J_ zW@$(;5G0UhYJ<|tYw(Pasw)#tvc$xmqw3p1he(#kD@YPg^vKoElB2J}B_vA)85s7+aq?O{xkNwj1Sk!llVlZ>dE`cuA`)`WIX-Xuq-* zc${sI!D_=W42I9qQy6mDLN2{Fr?L$Sg)-O!7|&K3vsNyWT_~mRUONpXf$1RQPyYS? zs(numd-e0R|J2``kDcB^oq;D@Fp$W}MkPp2pN;*~h7Y1{P}x8dl^qPLhS8$GV3~JN zB@%xibn1fq8wA&@7OHCe!5ojX+sQKreu6%*MV-#h#7U3@bsRj6wol~vsyWem)xo_3 z@H7kd_J7i*^sH55%|kns*s4jzcnT5ACgps+FAl_dK;xucsGE_!=M;4|^gTQP$w6%7 zybXAqUCloW!axiK@VWOX0=H=C(%H$apin`aL+#~ku;%10iuKc5vG{Mx`|)_O?8T9X zZM7_MUDvDK1}kGnVxd1ZbW@4~EDEV>N+i)Z;5npO8zd=Q13SpI+4oeKEXx2A-w73{ zrxQN!Up~@+!TUry`9%^RgdFst_=JwXg2bu&dinOYYa-<1;(TE&iDPdbdr(GJx*B+# zb&tVr!Y~Ym&-GJ8^fC}k+OB;7I7~tsyGNl(S4)i(r3nln-kn5*s-o+~vj2Sf{kz15 zq-xk>Vj-36)G!V2GRI_&*>y8!6D3$|c${U7QES356oudOD=ze9jtaHVz3eHonaZjuS%tlY*xX`}#*su3 zfBjm@YQeXA&iU>anZ{~t;Cqv^54y|cnnYbT7N^y7R|;P7B`+isf(w|Z&xl}KUa@~* z@sQ-~lkSU3mUQ{WD!HXUut<_*+zTpJY`VT>S(*@HYygM#ReGcGpsaqW&(;U1kIKPo zi!tJsvY9_5{3hvo4ph7%^hU`k5)fC{od$->x&qsX03vXVf?%g+^?!n7-Xw% zot8wM&otx@f9+{%w2lIJoQ+s*Z`(Ey{#^fxTk;`x;<#z{EonP+TZRVPnxV;nVqM|U z5@ivUNR6cIdPV>Ho+Bk&w&k=!QBx%E?z!h)Naa{%I+YpSKfL|%&tImm2gi6*T5cu= zPfrJQO84H*>g>c(A~ek;)lQPqR0_SKrC7Nu5{?ATR$h{@R;Rli$MQnjtT0P*B9qu=m7co^);H!H<F#lAvHks;rFlSYTrx zrK|!hrPHfM)w2mCB%Qy6LMmlBla>^)77QB>6opC7Wjdx@>LdW!bu@)))`VwPt0JYu zlvN==wVgLrN|<9KuTK!uz`0{9bC ztvYBcqqGNH0jQHk>YoU3no2K{{Df7ofT^SvUKt(fiK`(Z0UKr3l%&9(%VkkCx|X>r zWFQT01T|rm7NSroEv*j{8|qu|mj@{Agg?UI@ZqNb10LH8sp*WKg>A0E5cn5frW1OA zPWcIG67@i1Lw=h?bn^Gb$vB8CTf_lGqpN`eff?_PZv1o}!!)6rkT8)y%4(eizNB_F zl>1z@kk$?}gvv@M?E?5Vk$Ypjj1H=>=k8awc~li*bwo?$^MI^jK2=sG9`RQhtxQcz zV?RQ!Q5CcxyjL6TO@QatEQfA3ru)0!Z>IORf8C6(n;m#2lL@_+R*r)yIBwutWZrpU zJ<|itZWj0fXF4U5ArgA-^KCVaJ+Tm~fWc9$j)K8j7bO`tS6suC@ZX@J4f`7YWZOR{ zJ)-}-#V7mF1Ah>e>TjL zGYprT5OZnjdCrkiF?nMurABYM9ip5_<|R#=9Bi{sIma`v{hx$`WsJvxFf>GSLq0UcVw=@Dg;KV&nDT%k^OQ^tyd>`#|)WF2?+C+Z27_qwSoZ)>ff#5GAVwp$hYG zpn)!ILMuE&l?Rp2Uh7GUn$SPk5zun zZGw{++&a)8b_3Iw+BjOv!-~E_mXNarN+C`6 zD}jAvG)bGbTYNtoby#fJJv(DAll_G*#r+K;#O+W4R|ba%D8`YsU7UxSM~R#AjW?v4llGb}g)9EKYvG=P= z^7mXbJ3N)2DqP^EnFe|EHEzPyCO1VYZF_QrUhwHaZbs!_)0!Du3LHHhD3s}a1A{4_ zVph#+FQ?PtFv<;a@6bLQUyi>YjYbD}>W#-1h7tg9RqZ6;_Q8_jiq(^Z?XmgG?Qb`s zKcTq7T@5{)B`Cvr6fViQ81l6nm(DQHA1^xG8#i_4woUe1$v84@9n27J$)<;DfXZIpRGTYHKftN>(g`hBC1?k!nqTb zCFPPv*!cY$Dt;I?Lf|K3AW)tngBoOjnq}YdVZv63$RR0)tiVa!&E45OAMLGN>^fxclP%Ua14p};8F$(X6B`0h;yY@l%y8rDP-oADCFnm z0F}fi7nBxjl;vlpXj*e|DW~S8WTq+TYoY5@(9-7u0AB|oS+$MV(}5MSk9FXR(XP{eRA3NSrnC~qu8Z^bi_VM0a&B3<)wT!D(&I|~nqNC`jz~xoWjW&`GWyIZoqhYca;u?Lpa`~yc{LxF`R5tUnIZMWO@sqHyQl6PLfdqUpMLR zFKsHsi9)EImeG6UZ5D;%TuRWS~w)zaYG#_ z60Z?%64!!Wj)ue0&&lBX9dvV5Rch}1JbXmn^LdN6T`jPoSr3zzhc8?!P^i{(VfYb@ zHPQk$z#+^qLt2DsvKdxx?FH~)N5qq}qm<=VEIE2R9j<5SwFJEyz%k=?VkD00|A!qv zvhvEdr>!Hp<1W0***ANq-@3D~xC;KdFYy}bs5o`Az9W&7A7sqwF^c=d-!@@#)Duo`6B99XhM>3Kex-I+PJ0m%czd~Ry zaxycTdGC#8)~eDaR`BcR%S-Y5+mG+XYqBPT(m1ju&f8Xc(Bg~*YH#1*U$T-$I}fdP z>Y%X%c@!2hcNAZCS6?(W!XL@h2P*C1Y)afE@4G&=w}?fFPTn$Kj4C19w^fCf?$0rV zbDG(yzkO!ml_X=$d=}A$sv%8*f4mjar!zbmpE(Q}1xig0qrw?;YBmYskACb{9A)OpNuojdZ!e1y%+LS_)W?#J(cS$+-Yhnm5t^F zM&Z+YMj!&+s8eTUV(waZPS)Sc!{rPI^y(@1YRs7=uWBW| zqre6AprSjg*h+fQBCZY2ctAgV-ixgpuY`Q=?hPy5!z{}C=Z;DZl^l{<{W-lCWW_xt>78vhf;d~JCU;}aN_PJ{)e46ex9ep{_ zk8X5mDXmeLs!IL?thr|jyft{7eN#(sgD?y}lV4%AmlmN`J@+(iNZV~oMR;(+AQgwFq{N$|2lzMS~mA!Wj4Z6jCBM<77?72I0qY} zxYOktUCTr?iu~|U491lI1Ls71@ zUElx1A2dwjo4d3ZGKp_rlBb8)7*@iG1pU#(SNgt zBeK6IuG3#1%fHjcF5-NVkqCkS{4<2#NP|5nIIBV$Cc3b(Tf{M?u$G*a!^H8ZP|!7s z7l>W$(n>B(zjU2+H!0MI#rsC{=FRidl4@tM)<5pQ+)W!I6Ri2)WG5QEksY&{`T2yjUhOAB0#nyMN1;4?# zjl@{KtUAxo1@_jkNTl{+-BK)1k+(h}bh3=-K|4G0KyDb{noz1gXY?QT6!D!))Ht-g zdl3`zhj-Z?ouA&z$^R~7WfMTA;SRiO_Y=^05l0&`EzAjC;A!3{vLE{8=3`9Lh_pnT zDVQE9hz~uLo#2gp=uA6vCPc~CPQbiq@DiLu1jJ|V@lhwsIPP{Oa=E3sB;*c~ah8`$S*Cp$h>SAiw z@{kkUF-F(NV7lk(7!Y@@M~Ykki>cBD|I1a%+`N|Z*6fD}oVlTFBpt_bXLyCNC*1ni z=;Xq`jjO*s|AooBSC@Pfms+B;RI9(aSk$w$B6ysYQ9WzJFc97KD-OIg#t=xCK*mDb zKya3%t5D>#b2L61=}g-G`btXc#Ld)2x{r77J)JVDxxyM=);qo5JumgE%0f_FivjUl`E2!`4u+=X5_?sU#Wa&Q!_qpM&I+=449nL#}OP+f~8+J&sL_0hfTi z3#UFi(}#uV$Z$tlq?bo|p2Ukwazc(|+0Ww3tzJKE%Qtl)aF~3c_7aqu2{~~Aw|O_n z=z7{A9veX`!;{DHTmLGU;!xNbl>5ylAi1Ji{oa9$8&uzvro5ZIM|hl#R84Q&Fbq9M zzk+Bkju52SIj0^rY*?o)f9cxo0L!#0ZMgRLKe{{B+6}#BrBl#ZrBztQcZ?J(M zclX~vKd7%_%aQf?7Z6+PYnKheO~i8`JhV4LB#OBvc+&0{MtBjMORxr{(WH-JV?9A7 z8>G5b5u07LESGy20ZxJ}6~~dGk{ySo@k&$n$(}qKs9ViIYW1hxQtkhnMzz|Z_PKl4 zH}Rg36h*{Y*xE!PKF<*C+$_3ad<+TQydq*97(a>uszZ#qBNZNhVojZ&M@ zz<%PrDz?`8*1#JCHbE*K)M9_*8tc&OGXjO7s?Q9(aM3Q8%%KdnVp1-gltI$gr8ib> zj3pbqX8Z6rJ7w8O-F3)=v44{&w;onz_E&a&YWCc@J$$gZ*lhl>`_hxEQx|*YZuc%< zy*H(DmFRCjQKtr=nycmLW?yyrPm~Y4ybIA5y|E3jozs7%Sn&rZvxeKWFL<1FQqgW2 zF%W!?zhWg`k`qck;HpX&AfakSD{?50ma+G^RSx@PuMeuK{~mMX8lcI8CC_+vyq@)l zDr>NUr?MzlPjW6s36aHjLX61jCK`kS6ffdI9f5g>uDTl&Y zF`X^Pr_}1#v?yoOGj1|37LOO)YEdqW$@dF(@$-IJEGF}@_?rt6*hkH9b~#xF3X&`_ zsU{a!x>g>twO3Zx2k2~IIFcSK^k_Ba$GQ&v+JynBEn4%okCwgCFF0W9cFNlX?lLHA z3<%0&%3|9_Mgt~#vKwf<(-^{G>nh+Hfvlm9OjHd7)J&-v$nK7wsd`JS*3Hns&iNNZ zUZ%yrFSyf&PM?%6z`d-fH+lH>V+L8(UTE6%ZrbO4ga^+J4FSp|t7vn5s_v8@!#UAJ z97E^GK&IC>xxhW6x6t?Z&ngC#{pa}P++(AdI+U&AI^?4Fdp}N1wkgx=|7M(pn-9dv z`xB`cr7=tA6tONb&6tep_Fq6-6JV6&X$ z+*rhy$RAY;;=glF+$u#SLlVDt_nzO`nO4|<;vdi3%|>ir-`8)O$EUS;VHv?1;R?et zjdd3q;LAg!mAV_>x9yTK=WVDx9{~YId*QtFuwu*~2Z%=dUN<~1W)*dhE*KCBZ}O2X zN?y0p#&1BUcL)vddfeTt{x#s=Y;jeRr{5J@Oqv#q)D+nga-q)8jBxsk8h*%5-=J>r+Ln`W9%rD0a82r1>@JEXpwMKmy0)Z#7BX6oB<#J z-~XQypAxW>2o#vi;jlQV2)@M}%`bi&A7qTQDWQ|62zcsc4jgyb0GOF1L<-OAdy2-D tFDyW&#L@M5oLj;&mt})2695Vb0vUd~@0o~~*#qY0wst)_Zp7SMgj;>}a0>tc literal 0 HcmV?d00001 diff --git a/test/test_pack.py b/test/test_pack.py index a0c8464e0..199b6c61c 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -13,14 +13,13 @@ class TestPack(TestBase): - def test_pack_index(self): - # read v2 index information - index_file = fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx') - index = PackIndex(index_file) - + packindexfile_v2 = fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx') + packindexfile_v1 = fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx') + + def _assert_index_file(self, index, version, size): assert index.packfile_checksum != index.indexfile_checksum - assert index.version == 2 - assert index.size == 30 + assert index.version == version + assert index.size == size # get all data of all objects for oidx in xrange(index.size): @@ -35,4 +34,14 @@ def test_pack_index(self): assert entry[2] == index.crc(oidx) # END for each object index in indexfile + + def test_pack_index(self): + # check version 1 and 2 + index = PackIndex(self.packindexfile_v1) + self._assert_index_file(index, 1, 67) + + index = PackIndex(self.packindexfile_v2) + self._assert_index_file(index, 2, 30) + + From 3b902ed6bf75bb04bdf5703a564f539d8d2e43d8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Jun 2010 23:35:46 +0200 Subject: [PATCH 0014/1392] Initial version of a pack design that should be able to solve the problem nicely streams: added pack specific Info and Stream types, including test --- fun.py | 27 ++++---- pack.py | 147 ++++++++++++++++++++++++++++++++++++++++---- stream.py | 84 ++++++++++++++++++++++++- test/test_pack.py | 54 ++++++++++++---- test/test_stream.py | 28 +++++++++ 5 files changed, 298 insertions(+), 42 deletions(-) diff --git a/fun.py b/fun.py index 883062eba..b2e684472 100644 --- a/fun.py +++ b/fun.py @@ -11,11 +11,17 @@ # INVARIANTS +OFS_DELTA = 6 +REF_DELTA = 7 type_id_to_type_map = { + 0 : "", # EXT 1 1 : "commit", 2 : "tree", 3 : "blob", - 4 : "tag" + 4 : "tag", + 5 : "", # EXT 2 + OFS_DELTA : "OFS_DELTA", # OFFSET DELTA + REF_DELTA : "REF_DELTA" # REFERENCE DELTA } # used when dealing with larger streams @@ -42,30 +48,23 @@ def loose_object_header_info(m): type_name, size = hdr[:hdr.find("\0")].split(" ") return type_name, int(size) -def object_header_info(m): - """:return: tuple(type_string, uncompressed_size_in_bytes - :param mmap: mapped memory map. It will be - seeked to the actual start of the object contents, which can be used - to initialize a zlib decompress object. - :note: This routine can only handle new-style objects which are assumably contained - in packs - """ - assert not is_loose_object(m), "Use loose_object_header_info instead" - +def pack_object_header_info(data): + """:return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) + The type_id should be interpreted according to the ``type_id_to_type_map`` map + The byte-offset specifies the start of the actual zlib compressed datastream + :param m: random-access memory, like a string or memory map""" c = b0 # first byte i = 1 # next char to read type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size s = 4 # starting bit-shift size while c & 0x80: - c = ord(m[i]) + c = ord(data[i]) i += 1 size += (c & 0x7f) << s s += 7 # END character loop - # finally seek the map to the start of the data stream - m.seek(i) try: return (type_id_to_type_map[type_id], size) except KeyError: diff --git a/pack.py b/pack.py index 2ffc64f52..377963053 100644 --- a/pack.py +++ b/pack.py @@ -1,4 +1,4 @@ -"""Contains PackIndex and PackFile implementations""" +"""Contains PackIndexFile and PackFile implementations""" from util import ( LockedFD, LazyMixin, @@ -6,14 +6,17 @@ unpack_from ) +from fun import ( + pack_object_header_info + ) from struct import ( pack, ) -__all__ = ('PackIndex', 'Pack') +__all__ = ('PackIndexFile', 'PackFile') -class PackIndex(LazyMixin): +class PackIndexFile(LazyMixin): """A pack index provides offsets into the corresponding pack, allowing to find locations for offsets faster.""" @@ -26,7 +29,7 @@ class PackIndex(LazyMixin): _sha_list_offset = 8 + 1024 def __init__(self, indexpath): - super(PackIndex, self).__init__() + super(PackIndexFile, self).__init__() self._indexpath = indexpath def _set_cache_(self, attr): @@ -121,9 +124,9 @@ def _initialize(self): self._fanout_table = self._read_fanout((self._version == 2) * 8) if self._version == 2: - self._crc_list_offset = self._sha_list_offset + self.size * 20 - self._pack_offset = self._crc_list_offset + self.size * 4 - self._pack_64_offset = self._pack_offset + self.size * 4 + self._crc_list_offset = self._sha_list_offset + self.size() * 20 + self._pack_offset = self._crc_list_offset + self.size() * 4 + self._pack_64_offset = self._pack_offset + self.size() * 4 # END setup base def _read_fanout(self, byte_offset): @@ -139,21 +142,17 @@ def _read_fanout(self, byte_offset): #} END initialization #{ Properties - @property def version(self): return self._version - @property def size(self): """:return: amount of objects referred to by this index""" return self._fanout_table[255] - @property def packfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of the pack file""" return self._data[-40:-20] - @property def indexfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of this index file""" return self._data[-20:] @@ -186,6 +185,128 @@ def sha_to_index(self, sha): #} END properties -class Pack(LazyMixin): - """A pack is a file written according to the Version 2 for git packs""" +class PackFile(LazyMixin): + """A pack is a file written according to the Version 2 for git packs + As we currently use memory maps, it could be assumed that the maximum size of + packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be + fine though. + + :note: at some point, this might be implemented using streams as well, or + streams are an alternate path in the case memory maps cannot be created + for some reason - one clearly doesn't want to read 10GB at once in that + case""" + + __slots__ = ('_packpath', '_data', '_size', '_version') + + # offset into our data at which the first object starts + _first_object_offset = 3*4 + 8 + + def __init__(self, packpath): + self._packpath = packpath + + def _set_cache_(self, attr): + if attr == '_data': + ldb = LockedFD(self._packpath) + fd = ldb.open() + self._data = file_contents_ro(fd) + ldb.rollback() + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? + else: + # read the header information + type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0) + assert type_id == "PACK", "Pack file format is invalid: %r" % type_id + assert self._version in (2, 3), "Cannot handle pack format version %i" % self._version + # END handle header + + def _iter_objects(self, start_offset, as_stream): + """Handle the actual iteration of objects within this pack""" + size = len(self._data) + cur_offset = start_offset or self._first_object_offset + + while cur_offset < size: + type_id, uncomp_size, data_offset = pack_object_header_info(buffer(self._data, cur_offset)) + + # if type_id + # END until we have read everything + + #{ Interface + + def size(self): + """:return: The amount of objects stored in this pack""" + return self._size + + def version(self): + """:return: the version of this pack""" + return self._version + + def checksum(self): + """:return: 20 byte sha1 hash on all object sha's contained in this file""" + return self._data[-20:] + + #} END interface + + #{ Read-Database like Interface + + def info(self, offset): + """Retrieve information about the object at the given file-absolute offset + :param offset: byte offset + :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" + raise NotImplementedError() + + def stream(self, offset): + """Retrieve an object at the given file-relative offset as stream along with its information + :param offset: byte offset + :return: OPackStream instance, the actual type differs depending on the type_id attribute""" + raise NotImplementedError() + + #} END Read-Database like Interface + + +class PackFileEntity(object): + """Combines the PackIndexFile and the PackFile into one, allowing the + actual objects to be resolved and iterated""" + + __slots__ = ('_index', '_pack') + + IndexFileCls = PackIndexFile + PackFileCls = PackFile + + def __init__(self, basename): + self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance + self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + + + def _iter_objects(self, as_stream): + raise NotImplementedError + + #{ Read-Database like Interface + + def info(self, sha): + """Retrieve information about the object identified by the given sha + :param sha: 20 byte sha1 + :return: OInfo instance""" + raise NotImplementedError() + + def stream(self, sha): + """Retrieve an object stream along with its information as identified by the given sha + :param sha: 20 byte sha1 + :return: OStream instance""" + raise NotImplementedError() + + #} END Read-Database like Interface + + #{ Interface + + def info_iter(self): + """:return: Iterator over all objects in this pack. The iterator yields + OInfo instances""" + return self._iter_objects(as_stream=False) + + def stream_iter(self): + """:return: iterator over all objects in this pack. The iterator yields + OStream instances""" + return self._iter_objects(as_stream=True) + + #} Interface diff --git a/stream.py b/stream.py index 44c7b945a..b30ec1ec9 100644 --- a/stream.py +++ b/stream.py @@ -11,7 +11,11 @@ zlib ) -__all__ = ('OInfo', 'OStream', 'IStream', 'InvalidOInfo', 'InvalidOStream', +from fun import type_id_to_type_map + +__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', + 'OStream', 'OPackStream', 'ODeltaPackStream', + 'IStream', 'InvalidOInfo', 'InvalidOStream', 'DecompressMemMapReader', 'FDCompressedSha1Writer') @@ -55,8 +59,42 @@ def type(self): def size(self): return self[2] #} END interface - - + + +class OPackInfo(OInfo): + """As OInfo, but provides a type_id property to retrieve the numerical type id""" + __slots__ = tuple() + + @property + def type(self): + return type_id_to_type_map[self[1]] + + #{ Interface + + @property + def type_id(self): + return self[1] + + #} interface + + +class ODeltaPackInfo(OPackInfo): + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, + or the base_offset, being an offset into the pack at which our base + can be found""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, delta_info): + return tuple.__new__(cls, (sha, type, size, delta_info)) + + #{ Interface + @property + def delta_info(self): + return self[3] + #} END interface + + class OStream(OInfo): """Base for object streams retrieved from the database, providing additional information about the stream. @@ -76,6 +114,46 @@ def __init__(self, *args, **kwargs): def read(self, size=-1): return self[3].read(size) + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class OPackStream(OPackInfo): + """Next to pack object information, a stream outputting an undeltified base object + is provided""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + #{ Stream Reader Interface + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class ODeltaPackStream(ODeltaPackInfo): + """Provides a stream outputting the uncompressed offset delta information""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, delta_info, stream): + return tuple.__new__(cls, (sha, type, size, delta_info, stream)) + + + #{ Stream Reader Interface + def read(self, size=-1): + return self[4].read(size) + + @property + def stream(self): + return self[4] #} END stream reader interface diff --git a/test/test_pack.py b/test/test_pack.py index 199b6c61c..14d8a2496 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -6,23 +6,35 @@ fixture_path ) from gitdb.pack import ( - PackIndex + PackIndexFile, + PackFile ) +from gitdb.util import to_bin_sha import os +#{ Utilities +def bin_sha_from_filename(filename): + return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:]) +#} END utilities + class TestPack(TestBase): - packindexfile_v2 = fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx') - packindexfile_v1 = fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx') + packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) + packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) + packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) + packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) + def _assert_index_file(self, index, version, size): - assert index.packfile_checksum != index.indexfile_checksum - assert index.version == version - assert index.size == size + assert index.packfile_checksum() != index.indexfile_checksum() + assert len(index.packfile_checksum()) == 20 + assert len(index.indexfile_checksum()) == 20 + assert index.version() == version + assert index.size() == size # get all data of all objects - for oidx in xrange(index.size): + for oidx in xrange(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) @@ -34,14 +46,32 @@ def _assert_index_file(self, index, version, size): assert entry[2] == index.crc(oidx) # END for each object index in indexfile + + def _assert_pack_file(self, pack, version, size): + assert pack.version() == 2 + assert pack.size() == size + assert len(pack.checksum()) == 20 + def test_pack_index(self): # check version 1 and 2 - index = PackIndex(self.packindexfile_v1) - self._assert_index_file(index, 1, 67) - - index = PackIndex(self.packindexfile_v2) - self._assert_index_file(index, 2, 30) + for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): + index = PackIndexFile(indexfile) + self._assert_index_file(index, version, size) + # END run tests + def test_pack(self): + # there is this special version 3, but apparently its like 2 ... + for packfile, version, size in (self.packfile_v2_1, self.packfile_v2_2): + pack = PackFile(packfile) + self._assert_pack_file(pack, version, size) + # END for each pack to test + def test_pack_entity(self): + # TODO: + pass + def test_pack_64(self): + # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets + # of course without really needing such a huge pack + pass diff --git a/test/test_stream.py b/test/test_stream.py index 4f022286e..7c9097961 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -39,15 +39,43 @@ def test_streams(self): assert info.type == str_blob_type assert info.size == s + # test pack info + # provides type_id + blob_id = 3 + pinfo = OPackInfo(sha, blob_id, s) + assert pinfo.type == str_blob_type + assert pinfo.type_id == blob_id + + dpinfo = ODeltaPackInfo(sha, blob_id, s, sha) + assert dpinfo.type == str_blob_type + assert dpinfo.type_id == blob_id + assert dpinfo.delta_info == sha + + # test ostream stream = DummyStream() ostream = OStream(*(info + (stream, ))) + assert ostream.stream is stream ostream.read(15) stream._assert() assert stream.bytes == 15 ostream.read(20) assert stream.bytes == 20 + # test packstream + postream = OPackStream(*(pinfo + (stream, ))) + assert postream.stream is stream + postream.read(10) + stream._assert() + assert stream.bytes == 10 + + # test deltapackstream + dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) + dpostream.stream is stream + dpostream.read(5) + stream._assert() + assert stream.bytes == 5 + # derive with own args DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() From 0650892246e99b60448d4a168ea36f84236b97a4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 01:10:46 +0200 Subject: [PATCH 0015/1392] moved all info and stream base classes into new module, base, as well as the respective tests were moved to test_base Adjusted PackStream and PackInfo classes not to contain the sha field anymore streams: DecompressMemMapReader now parses its header on demand if it is not set, using the mose useful 3 lines ever, LazyMixin --- __init__.py | 1 + base.py | 272 +++++++++++++++++++++++++++++++++++++++++++ db/git.py | 2 +- db/loose.py | 9 +- pack.py | 76 +++++++++++- stream.py | 273 ++------------------------------------------ test/db/lib.py | 5 +- test/test_base.py | 90 +++++++++++++++ test/test_stream.py | 78 +------------ 9 files changed, 461 insertions(+), 345 deletions(-) create mode 100644 base.py create mode 100644 test/test_base.py diff --git a/__init__.py b/__init__.py index 8b0e47b19..d79788fc4 100644 --- a/__init__.py +++ b/__init__.py @@ -14,5 +14,6 @@ def _init_externals(): # default imports from db import * +from base import * from stream import * diff --git a/base.py b/base.py new file mode 100644 index 000000000..d12181229 --- /dev/null +++ b/base.py @@ -0,0 +1,272 @@ +"""Module with basic data structures - they are designed to be lightweight and fast""" +from util import ( + to_hex_sha, + to_bin_sha, + zlib + ) + +from fun import type_id_to_type_map + +__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', + 'OStream', 'OPackStream', 'ODeltaPackStream', + 'IStream', 'InvalidOInfo', 'InvalidOStream' ) + +#{ ODB Bases + +class OInfo(tuple): + """Carries information about an object in an ODB, provdiing information + about the sha of the object, the type_string as well as the uncompressed size + in bytes. + + It can be accessed using tuple notation and using attribute access notation:: + + assert dbi[0] == dbi.sha + assert dbi[1] == dbi.type + assert dbi[2] == dbi.size + + The type is designed to be as lighteight as possible.""" + __slots__ = tuple() + + def __new__(cls, sha, type, size): + return tuple.__new__(cls, (sha, type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + @property + def sha(self): + return self[0] + + @property + def type(self): + return self[1] + + @property + def size(self): + return self[2] + #} END interface + + +class OPackInfo(tuple): + """As OInfo, but provides a type_id property to retrieve the numerical type id, and + does not include a sha""" + __slots__ = tuple() + + def __new__(cls, type, size): + return tuple.__new__(cls, (type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + + @property + def type(self): + return type_id_to_type_map[self[0]] + + @property + def type_id(self): + return self[0] + + @property + def size(self): + return self[1] + + #} END interface + + +class ODeltaPackInfo(OPackInfo): + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, + or the base_offset, being an offset into the pack at which our base + can be found""" + __slots__ = tuple() + + def __new__(cls, type, size, delta_info): + return tuple.__new__(cls, (type, size, delta_info)) + + #{ Interface + @property + def delta_info(self): + return self[2] + #} END interface + + +class OStream(OInfo): + """Base for object streams retrieved from the database, providing additional + information about the stream. + Generally, ODB streams are read-only as objects are immutable""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + + def __init__(self, *args, **kwargs): + tuple.__init__(self) + + #{ Stream Reader Interface + + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class OPackStream(OPackInfo): + """Next to pack object information, a stream outputting an undeltified base object + is provided""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, *args): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (type, size, stream)) + + #{ Stream Reader Interface + def read(self, size=-1): + return self[2].read(size) + + @property + def stream(self): + return self[2] + #} END stream reader interface + + +class ODeltaPackStream(ODeltaPackInfo): + """Provides a stream outputting the uncompressed offset delta information""" + __slots__ = tuple() + + def __new__(cls, type, size, delta_info, stream): + return tuple.__new__(cls, (type, size, delta_info, stream)) + + + #{ Stream Reader Interface + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface + + +class IStream(list): + """Represents an input content stream to be fed into the ODB. It is mutable to allow + the ODB to record information about the operations outcome right in this instance. + + It provides interfaces for the OStream and a StreamReader to allow the instance + to blend in without prior conversion. + + The only method your content stream must support is 'read'""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, sha=None): + return list.__new__(cls, (sha, type, size, stream, None)) + + def __init__(self, type, size, stream, sha=None): + list.__init__(self, (sha, type, size, stream, None)) + + #{ Interface + + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return to_hex_sha(self[0]) + + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return to_bin_sha(self[0]) + + def _error(self): + """:return: the error that occurred when processing the stream, or None""" + return self[4] + + def _set_error(self, exc): + """Set this input stream to the given exc, may be None to reset the error""" + self[4] = exc + + error = property(_error, _set_error) + + #} END interface + + #{ Stream Reader Interface + + def read(self, size=-1): + """Implements a simple stream reader interface, passing the read call on + to our internal stream""" + return self[3].read(size) + + #} END stream reader interface + + #{ interface + + def _set_sha(self, sha): + self[0] = sha + + def _sha(self): + return self[0] + + sha = property(_sha, _set_sha) + + + def _type(self): + return self[1] + + def _set_type(self, type): + self[1] = type + + type = property(_type, _set_type) + + def _size(self): + return self[2] + + def _set_size(self, size): + self[2] = size + + size = property(_size, _set_size) + + def _stream(self): + return self[3] + + def _set_stream(self, stream): + self[3] = stream + + stream = property(_stream, _set_stream) + + #} END odb info interface + + +class InvalidOInfo(tuple): + """Carries information about a sha identifying an object which is invalid in + the queried database. The exception attribute provides more information about + the cause of the issue""" + __slots__ = tuple() + + def __new__(cls, sha, exc): + return tuple.__new__(cls, (sha, exc)) + + def __init__(self, sha, exc): + tuple.__init__(self, (sha, exc)) + + @property + def sha(self): + return self[0] + + @property + def error(self): + """:return: exception instance explaining the failure""" + return self[1] + + +class InvalidOStream(InvalidOInfo): + """Carries information about an invalid ODB stream""" + __slots__ = tuple() + +#} END ODB Bases + diff --git a/db/git.py b/db/git.py index d2477d7b1..0488bc601 100644 --- a/db/git.py +++ b/db/git.py @@ -1,5 +1,5 @@ -from gitdb.stream import ( +from gitdb.base import ( OInfo, OStream ) diff --git a/db/loose.py b/db/loose.py index 37aad8c6f..109782fdc 100644 --- a/db/loose.py +++ b/db/loose.py @@ -13,11 +13,14 @@ from gitdb.stream import ( DecompressMemMapReader, FDCompressedSha1Writer, - Sha1Writer, - OStream, - OInfo + Sha1Writer ) +from gitdb.base import ( + OStream, + OInfo + ) + from gitdb.util import ( ENOENT, to_hex_sha, diff --git a/pack.py b/pack.py index 377963053..ab394daba 100644 --- a/pack.py +++ b/pack.py @@ -7,8 +7,21 @@ ) from fun import ( - pack_object_header_info + pack_object_header_info, + OFS_DELTA, + REF_DELTA ) + +from base import ( + OPackInfo, + OPackStream, + ODeltaPackInfo, + ODeltaPackStream, + ) +from stream import ( + DecompressMemMapReader, + ) + from struct import ( pack, ) @@ -16,6 +29,60 @@ __all__ = ('PackIndexFile', 'PackFile') + +#{ Utilities + +def pack_object_at(data, as_stream): + """ + :return: info or stream object of the correct type according to the type + of the object, REF_DELTAS will not be resolved in case a stream is desired. + The resulting ODeltaPackStream will have None instead of a stream. + :param data: random accessable data at which the header of an object can be read + :param as_stream: if True, a stream object will be returned that can read + the data, otherwise you receive an info object only + :note: a bit redundant, but it needs to be as fast as possible !""" + type_id, uncomp_size, data_offset = pack_object_header_info(data) + + if type_id == OFS_DELTA: + i = 0 + delta_offset = 0 + s = 7 + while c & 0x80: + c = ord(data[i]) + i += 1 + delta_offset += (c & 0x7f) << s + s += 7 + # END character loop + if as_stream: + stream = DecompressMemMapReader(buffer(data, i), False) + return ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) + else: + return ODeltaPackInfo(type_id, uncomp_size, delta_offset) + # END handle stream + elif type_id == REF_DELTA: + ref_sha = data[:20] + if as_stream: + stream = DecompressMemMapReader(buffer(data, 20), False) + return ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) + else: + return ODeltaPackInfo(type_id, uncomp_size, ref_sha) + # END handle stream + else: + # assume its a base object + if as_stream: + # if no size is given, it will read the header on first access + stream = DecompressMemMapReader(buffer(data, data_offset), False) + return OPackStream(type_id, uncomp_size, stream) + else: + return OPackInfo(type_id, uncomp_size) + # END handle as_stream + # END handle type id + + +#} END utilities + + + class PackIndexFile(LazyMixin): """A pack index provides offsets into the corresponding pack, allowing to find locations for offsets faster.""" @@ -222,13 +289,14 @@ def _set_cache_(self, attr): def _iter_objects(self, start_offset, as_stream): """Handle the actual iteration of objects within this pack""" - size = len(self._data) + data = self._data + size = len(data) cur_offset = start_offset or self._first_object_offset while cur_offset < size: - type_id, uncomp_size, data_offset = pack_object_header_info(buffer(self._data, cur_offset)) + ostream = pack_object_at(buffer(data, cur_offset), True) + # TODO: Decompressor needs to track the size of bytes actually decompressed - # if type_id # END until we have read everything #{ Interface diff --git a/stream.py b/stream.py index b30ec1ec9..f759267a5 100644 --- a/stream.py +++ b/stream.py @@ -3,279 +3,23 @@ import errno from util import ( - to_hex_sha, - to_bin_sha, + LazyMixin, make_sha, write, close, zlib ) -from fun import type_id_to_type_map - -__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', - 'OStream', 'OPackStream', 'ODeltaPackStream', - 'IStream', 'InvalidOInfo', 'InvalidOStream', - 'DecompressMemMapReader', 'FDCompressedSha1Writer') +__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer') # ZLIB configuration # used when compressing objects - 1 to 9 ( slowest ) Z_BEST_SPEED = 1 - -#{ ODB Bases - -class OInfo(tuple): - """Carries information about an object in an ODB, provdiing information - about the sha of the object, the type_string as well as the uncompressed size - in bytes. - - It can be accessed using tuple notation and using attribute access notation:: - - assert dbi[0] == dbi.sha - assert dbi[1] == dbi.type - assert dbi[2] == dbi.size - - The type is designed to be as lighteight as possible.""" - __slots__ = tuple() - - def __new__(cls, sha, type, size): - return tuple.__new__(cls, (sha, type, size)) - - def __init__(self, *args): - tuple.__init__(self) - - #{ Interface - @property - def sha(self): - return self[0] - - @property - def type(self): - return self[1] - - @property - def size(self): - return self[2] - #} END interface - - -class OPackInfo(OInfo): - """As OInfo, but provides a type_id property to retrieve the numerical type id""" - __slots__ = tuple() - - @property - def type(self): - return type_id_to_type_map[self[1]] - - #{ Interface - - @property - def type_id(self): - return self[1] - - #} interface - - -class ODeltaPackInfo(OPackInfo): - """Adds delta specific information, - Either the 20 byte sha which points to some object in the database, - or the base_offset, being an offset into the pack at which our base - can be found""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, delta_info): - return tuple.__new__(cls, (sha, type, size, delta_info)) - - #{ Interface - @property - def delta_info(self): - return self[3] - #} END interface - - -class OStream(OInfo): - """Base for object streams retrieved from the database, providing additional - information about the stream. - Generally, ODB streams are read-only as objects are immutable""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, stream, *args, **kwargs): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (sha, type, size, stream)) - - - def __init__(self, *args, **kwargs): - tuple.__init__(self) - - #{ Stream Reader Interface - - def read(self, size=-1): - return self[3].read(size) - - @property - def stream(self): - return self[3] - #} END stream reader interface - - -class OPackStream(OPackInfo): - """Next to pack object information, a stream outputting an undeltified base object - is provided""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, stream, *args): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (sha, type, size, stream)) - - #{ Stream Reader Interface - def read(self, size=-1): - return self[3].read(size) - - @property - def stream(self): - return self[3] - #} END stream reader interface - - -class ODeltaPackStream(ODeltaPackInfo): - """Provides a stream outputting the uncompressed offset delta information""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, delta_info, stream): - return tuple.__new__(cls, (sha, type, size, delta_info, stream)) - - - #{ Stream Reader Interface - def read(self, size=-1): - return self[4].read(size) - - @property - def stream(self): - return self[4] - #} END stream reader interface - - -class IStream(list): - """Represents an input content stream to be fed into the ODB. It is mutable to allow - the ODB to record information about the operations outcome right in this instance. - - It provides interfaces for the OStream and a StreamReader to allow the instance - to blend in without prior conversion. - - The only method your content stream must support is 'read'""" - __slots__ = tuple() - - def __new__(cls, type, size, stream, sha=None): - return list.__new__(cls, (sha, type, size, stream, None)) - - def __init__(self, type, size, stream, sha=None): - list.__init__(self, (sha, type, size, stream, None)) - - #{ Interface - - @property - def hexsha(self): - """:return: our sha, hex encoded, 40 bytes""" - return to_hex_sha(self[0]) - - @property - def binsha(self): - """:return: our sha as binary, 20 bytes""" - return to_bin_sha(self[0]) - - def _error(self): - """:return: the error that occurred when processing the stream, or None""" - return self[4] - - def _set_error(self, exc): - """Set this input stream to the given exc, may be None to reset the error""" - self[4] = exc - - error = property(_error, _set_error) - - #} END interface - - #{ Stream Reader Interface - - def read(self, size=-1): - """Implements a simple stream reader interface, passing the read call on - to our internal stream""" - return self[3].read(size) - - #} END stream reader interface - - #{ interface - - def _set_sha(self, sha): - self[0] = sha - - def _sha(self): - return self[0] - - sha = property(_sha, _set_sha) - - - def _type(self): - return self[1] - - def _set_type(self, type): - self[1] = type - - type = property(_type, _set_type) - - def _size(self): - return self[2] - - def _set_size(self, size): - self[2] = size - - size = property(_size, _set_size) - - def _stream(self): - return self[3] - - def _set_stream(self, stream): - self[3] = stream - - stream = property(_stream, _set_stream) - - #} END odb info interface - - -class InvalidOInfo(tuple): - """Carries information about a sha identifying an object which is invalid in - the queried database. The exception attribute provides more information about - the cause of the issue""" - __slots__ = tuple() - - def __new__(cls, sha, exc): - return tuple.__new__(cls, (sha, exc)) - - def __init__(self, sha, exc): - tuple.__init__(self, (sha, exc)) - - @property - def sha(self): - return self[0] - - @property - def error(self): - """:return: exception instance explaining the failure""" - return self[1] - - -class InvalidOStream(InvalidOInfo): - """Carries information about an invalid ODB stream""" - __slots__ = tuple() - -#} END ODB Bases - - #{ RO Streams -class DecompressMemMapReader(object): +class DecompressMemMapReader(LazyMixin): """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand buffered decompression accordingly @@ -296,19 +40,26 @@ class DecompressMemMapReader(object): max_read_size = 512*1024 # currently unused - def __init__(self, m, close_on_deletion, size): + def __init__(self, m, close_on_deletion, size=None): """Initialize with mmap for stream reading :param m: must be content data - use new if you have object data and no size""" self._m = m self._zip = zlib.decompressobj() self._buf = None # buffer of decompressed bytes self._buflen = 0 # length of bytes in buffer - self._s = size # size of uncompressed data to read in total + if size is not None: + self._s = size # size of uncompressed data to read in total self._br = 0 # num uncompressed bytes read self._cws = 0 # start byte of compression window self._cwe = 0 # end byte of compression window self._close = close_on_deletion # close the memmap on deletion ? + def _set_cache_(self, attr): + assert attr == '_s' + # only happens for size, which is a marker to indicate we still + # have to parse the header from the stream + self._parse_header_info() + def __del__(self): if self._close: self._m.close() diff --git a/test/db/lib.py b/test/db/lib.py index 35823059b..cf752741b 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -6,8 +6,9 @@ TestBase ) -from gitdb.stream import ( - Sha1Writer, +from gitdb.stream import Sha1Writer + +from gitdb.base import ( IStream, OStream, OInfo diff --git a/test/test_base.py b/test/test_base.py new file mode 100644 index 000000000..f7ddebaa2 --- /dev/null +++ b/test/test_base.py @@ -0,0 +1,90 @@ +"""Test for object db""" +from lib import ( + TestBase, + DummyStream, + DeriveTest, + ) + +from gitdb import * +from gitdb.util import ( + NULL_HEX_SHA + ) + +from gitdb.typ import ( + str_blob_type + ) + + +class TestBaseTypes(TestBase): + + def test_streams(self): + # test info + sha = NULL_HEX_SHA + s = 20 + info = OInfo(sha, str_blob_type, s) + assert info.sha == sha + assert info.type == str_blob_type + assert info.size == s + + # test pack info + # provides type_id + blob_id = 3 + pinfo = OPackInfo(blob_id, s) + assert pinfo.type == str_blob_type + assert pinfo.type_id == blob_id + + dpinfo = ODeltaPackInfo(blob_id, s, sha) + assert dpinfo.type == str_blob_type + assert dpinfo.type_id == blob_id + assert dpinfo.delta_info == sha + + + # test ostream + stream = DummyStream() + ostream = OStream(*(info + (stream, ))) + assert ostream.stream is stream + ostream.read(15) + stream._assert() + assert stream.bytes == 15 + ostream.read(20) + assert stream.bytes == 20 + + # test packstream + postream = OPackStream(*(pinfo + (stream, ))) + assert postream.stream is stream + postream.read(10) + stream._assert() + assert stream.bytes == 10 + + # test deltapackstream + dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) + dpostream.stream is stream + dpostream.read(5) + stream._assert() + assert stream.bytes == 5 + + # derive with own args + DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() + + # test istream + istream = IStream(str_blob_type, s, stream) + assert istream.sha == None + istream.sha = sha + assert istream.sha == sha + + assert len(istream.binsha) == 20 + assert len(istream.hexsha) == 40 + + assert istream.size == s + istream.size = s * 2 + istream.size == s * 2 + assert istream.type == str_blob_type + istream.type = "something" + assert istream.type == "something" + assert istream.stream is stream + istream.stream = None + assert istream.stream is None + + assert istream.error is None + istream.error = Exception() + assert isinstance(istream.error, Exception) diff --git a/test/test_stream.py b/test/test_stream.py index 7c9097961..69a93ad0f 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -2,7 +2,6 @@ from lib import ( TestBase, DummyStream, - DeriveTest, Sha1Writer, make_bytes, make_object @@ -18,7 +17,6 @@ str_blob_type ) -from cStringIO import StringIO import tempfile import os @@ -29,78 +27,6 @@ class TestStream(TestBase): """Test stream classes""" data_sizes = (15, 10000, 1000*1024+512) - - def test_streams(self): - # test info - sha = NULL_HEX_SHA - s = 20 - info = OInfo(sha, str_blob_type, s) - assert info.sha == sha - assert info.type == str_blob_type - assert info.size == s - - # test pack info - # provides type_id - blob_id = 3 - pinfo = OPackInfo(sha, blob_id, s) - assert pinfo.type == str_blob_type - assert pinfo.type_id == blob_id - - dpinfo = ODeltaPackInfo(sha, blob_id, s, sha) - assert dpinfo.type == str_blob_type - assert dpinfo.type_id == blob_id - assert dpinfo.delta_info == sha - - - # test ostream - stream = DummyStream() - ostream = OStream(*(info + (stream, ))) - assert ostream.stream is stream - ostream.read(15) - stream._assert() - assert stream.bytes == 15 - ostream.read(20) - assert stream.bytes == 20 - - # test packstream - postream = OPackStream(*(pinfo + (stream, ))) - assert postream.stream is stream - postream.read(10) - stream._assert() - assert stream.bytes == 10 - - # test deltapackstream - dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) - dpostream.stream is stream - dpostream.read(5) - stream._assert() - assert stream.bytes == 5 - - # derive with own args - DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() - - # test istream - istream = IStream(str_blob_type, s, stream) - assert istream.sha == None - istream.sha = sha - assert istream.sha == sha - - assert len(istream.binsha) == 20 - assert len(istream.hexsha) == 40 - - assert istream.size == s - istream.size = s * 2 - istream.size == s * 2 - assert istream.type == str_blob_type - istream.type = "something" - assert istream.type == "something" - assert istream.stream is stream - istream.stream = None - assert istream.stream is None - - assert istream.error is None - istream.error = Exception() - assert isinstance(istream.error, Exception) def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): """Make stream tests - the orig_stream is seekable, allowing it to be @@ -145,6 +71,10 @@ def test_decompress_reader(self): type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) assert size == len(cdata) assert type == str_blob_type + + # even if we don't set the size, it will be set automatically on first read + test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) + assert test_reader._s == len(cdata) else: # here we need content data zdata = zlib.compress(cdata) From bf4437ef45d9115aa2716e1c722c3938b6976803 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 13:37:59 +0200 Subject: [PATCH 0016/1392] DecompressMemMapReader: implemented compressed bytes counting, including test. This is required to properly read packs without the use of an index --- ext/async | 2 +- pack.py | 6 +-- stream.py | 105 +++++++++++++++++++++++++++++--------------- test/test_stream.py | 19 ++++---- 4 files changed, 82 insertions(+), 50 deletions(-) diff --git a/ext/async b/ext/async index af0040b0f..796b5e94f 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit af0040b0f3c6ede3be5b2d6bc69f6ea5ac53c36c +Subproject commit 796b5e94f19dfc36a3fb251468192373c76510b0 diff --git a/pack.py b/pack.py index ab394daba..2c01f0452 100644 --- a/pack.py +++ b/pack.py @@ -54,7 +54,7 @@ def pack_object_at(data, as_stream): s += 7 # END character loop if as_stream: - stream = DecompressMemMapReader(buffer(data, i), False) + stream = DecompressMemMapReader(buffer(data, i), False, uncomp_size) return ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) else: return ODeltaPackInfo(type_id, uncomp_size, delta_offset) @@ -62,7 +62,7 @@ def pack_object_at(data, as_stream): elif type_id == REF_DELTA: ref_sha = data[:20] if as_stream: - stream = DecompressMemMapReader(buffer(data, 20), False) + stream = DecompressMemMapReader(buffer(data, 20), False, uncomp_size) return ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) else: return ODeltaPackInfo(type_id, uncomp_size, ref_sha) @@ -267,7 +267,7 @@ class PackFile(LazyMixin): __slots__ = ('_packpath', '_data', '_size', '_version') # offset into our data at which the first object starts - _first_object_offset = 3*4 + 8 + _first_object_offset = 3*4 def __init__(self, packpath): self._packpath = packpath diff --git a/stream.py b/stream.py index f759267a5..de7ddcd86 100644 --- a/stream.py +++ b/stream.py @@ -1,6 +1,8 @@ from cStringIO import StringIO import errno +import mmap +import os from util import ( LazyMixin, @@ -13,10 +15,6 @@ __all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer') -# ZLIB configuration -# used when compressing objects - 1 to 9 ( slowest ) -Z_BEST_SPEED = 1 - #{ RO Streams class DecompressMemMapReader(LazyMixin): @@ -36,7 +34,8 @@ class DecompressMemMapReader(LazyMixin): times we actually allocate. An own zlib implementation would be good here to better support streamed reading - it would only need to keep the mmap and decompress it into chunks, thats all ... """ - __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close') + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', + '_cbr', '_phi') max_read_size = 512*1024 # currently unused @@ -52,6 +51,8 @@ def __init__(self, m, close_on_deletion, size=None): self._br = 0 # num uncompressed bytes read self._cws = 0 # start byte of compression window self._cwe = 0 # end byte of compression window + self._cbr = 0 # number of compressed bytes read + self._phi = False # is True if we parsed the header info self._close = close_on_deletion # close the memmap on deletion ? def _set_cache_(self, attr): @@ -85,6 +86,8 @@ def _parse_header_info(self): self._buf = StringIO(hdr[hdrend:]) self._buflen = len(hdr) - hdrend + self._phi = True + return type, size @classmethod @@ -98,7 +101,55 @@ def new(self, m, close_on_deletion=False): inst = DecompressMemMapReader(m, close_on_deletion, 0) type, size = inst._parse_header_info() return type, size, inst + + def compressed_bytes_read(self): + """:return: number of compressed bytes read. This includes the bytes it + took to decompress the header ( if there was one )""" + # ABSTRACT: When decompressing a byte stream, it can be that the first + # x bytes which were requested match the first x bytes in the loosely + # compressed datastream. This is the worst-case assumption that the reader + # does, it assumes that it will get at least X bytes from X compressed bytes + # in call cases. + # The caveat is that the object, according to our known uncompressed size, + # is already complete, but there are still some bytes left in the compressed + # stream that contribute to the amount of compressed bytes. + # How can we know that we are truly done, and have read all bytes we need + # to read ? + # Without help, we cannot know, as we need to obtain the status of the + # decompression. If it is not finished, we need to decompress more data + # until it is finished, to yield the actual number of compressed bytes + # belonging to the decompressed object + # We are using a custom zlib module for this, if its not present, + # we can only hope it works. + # Only scrub the stream forward if we are officially done with the + # bytes we were to have. + if self._br == self._s and hasattr(self._zip, 'status') and self._zip.status == zlib.Z_OK: + # manipulate the bytes-read to allow our own read method to coninute + # but keep the window at its current position + self._br = 0 + while self._zip.status == zlib.Z_OK: + self.read(mmap.PAGESIZE) + # END scrub-loop + # reset bytes read, just to be sure + self._br = self._s + # END handle stream scrubbing + + return self._cbr - len(self._zip.unused_data) + def seek(self, offset, whence=os.SEEK_SET): + """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != os.SEEK_SET: + raise ValueError("Can only seek to position 0") + # END handle offset + + self._zip = zlib.decompressobj() + self._br = self._cws = self._cwe = self._cbr = 0 + if self._phi: + self._phi = False + del(self._s) # trigger header parsing on first access + # END skip header + def read(self, size=-1): if size < 1: size = self._s - self._br @@ -109,33 +160,8 @@ def read(self, size=-1): if size == 0: return str() # END handle depletion - - # protect from memory peaks - # If he tries to read large chunks, our memory patterns get really bad - # as we end up copying a possibly huge chunk from our memory map right into - # memory. This might not even be possible. Nonetheless, try to dampen the - # effect a bit by reading in chunks, returning a huge string in the end. - # Our performance now depends on StringIO. This way we don't need two large - # buffers in peak times, but only one large one in the end which is - # the return buffer - # NO: We don't do it - if the user thinks its best, he is right. If he - # has trouble, he will start reading in chunks. According to our tests - # its still faster if we read 10 Mb at once instead of chunking it. - - # if size > self.max_read_size: - # sio = StringIO() - # while size: - # read_size = min(self.max_read_size, size) - # data = self.read(read_size) - # sio.write(data) - # size -= len(data) - # if len(data) < read_size: - # break - # # END data loop - # sio.seek(0) - # return sio.getvalue() - # # END handle maxread - # + + # deplete the buffer, then just continue using the decompress object # which has an own buffer. We just need this to transparently parse the # header from the zlib stream @@ -186,8 +212,7 @@ def read(self, size=-1): # if window is too small, make it larger so zip can decompress something - win_size = self._cwe - self._cws - if win_size < 8: + if self._cwe - self._cws < 8: self._cwe = self._cws + 8 # END adjust winsize @@ -196,10 +221,18 @@ def read(self, size=-1): # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) - + dcompdat = self._zip.decompress(indata, size) + # update the amount of compressed bytes read + # We feed possibly overlapping chunks, which is why the unconsumed tail + # has to be taken into consideration, as well as the unused data + # if we hit the end of the stream + self._cbr += len(indata) - len(self._zip.unconsumed_tail) self._br += len(dcompdat) + + print size, self._br, self._cbr, len(indata), self._cws, self._cwe, len(self._zip.unused_data), len(self._zip.unconsumed_tail) + if dat: dcompdat = dat + dcompdat @@ -252,7 +285,7 @@ class FDCompressedSha1Writer(Sha1Writer): def __init__(self, fd): super(FDCompressedSha1Writer, self).__init__() self.fd = fd - self.zip = zlib.compressobj(Z_BEST_SPEED) + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) #{ Stream Interface diff --git a/test/test_stream.py b/test/test_stream.py index 69a93ad0f..41f2b235a 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -49,12 +49,20 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): assert rest == cdata[-len(rest):] # END handle rest + if isinstance(stream, DecompressMemMapReader): + assert len(stream._m) == stream.compressed_bytes_read() + # END handle special type + rewind_stream(stream) # read everything rdata = stream.read() assert rdata == cdata + if isinstance(stream, DecompressMemMapReader): + assert len(stream._m) == stream.compressed_bytes_read() + # END handle special type + def test_decompress_reader(self): for close_on_deletion in range(2): for with_size in range(2): @@ -82,15 +90,7 @@ def test_decompress_reader(self): assert reader._s == len(cdata) # END get reader - def rewind(r): - r._zip = zlib.decompressobj() - r._br = r._cws = r._cwe = 0 - if with_size: - r._parse_header_info() - # END skip header - # END make rewind func - - self._assert_stream_reader(reader, cdata, rewind) + self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) # put in a dummy stream for closing dummy = DummyStream() @@ -99,7 +99,6 @@ def rewind(r): assert not dummy.closed del(reader) assert dummy.closed == close_on_deletion - #zdi# # END for each datasize # END whether size should be used # END whether stream should be closed when deleted From b6db082874b528b431d90de898a3061b2b6d9c36 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 14:01:45 +0200 Subject: [PATCH 0017/1392] DecompressMemMapReader: improved compressed_bytes_read method with alternate route which is a bit less efficient, but works without a custom zlib --- stream.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/stream.py b/stream.py index de7ddcd86..fb58125bf 100644 --- a/stream.py +++ b/stream.py @@ -120,16 +120,26 @@ def compressed_bytes_read(self): # until it is finished, to yield the actual number of compressed bytes # belonging to the decompressed object # We are using a custom zlib module for this, if its not present, - # we can only hope it works. + # we try to put in additional bytes up for decompression if feasible + # and check for the unused_data. + # Only scrub the stream forward if we are officially done with the # bytes we were to have. - if self._br == self._s and hasattr(self._zip, 'status') and self._zip.status == zlib.Z_OK: + if self._br == self._s and not self._zip.unused_data: # manipulate the bytes-read to allow our own read method to coninute # but keep the window at its current position self._br = 0 - while self._zip.status == zlib.Z_OK: - self.read(mmap.PAGESIZE) - # END scrub-loop + if hasattr(self._zip, 'status'): + while self._zip.status == zlib.Z_OK: + self.read(mmap.PAGESIZE) + # END scrub-loop custom zlib + else: + # pass in additional pages, until we have unused data + while not self._zip.unused_data and self._cbr != len(self._m): + self.read(mmap.PAGESIZE) + # END scrub-loop default zlib + # END handle stream scrubbing + # reset bytes read, just to be sure self._br = self._s # END handle stream scrubbing @@ -231,8 +241,6 @@ def read(self, size=-1): self._cbr += len(indata) - len(self._zip.unconsumed_tail) self._br += len(dcompdat) - print size, self._br, self._cbr, len(indata), self._cws, self._cwe, len(self._zip.unused_data), len(self._zip.unconsumed_tail) - if dat: dcompdat = dat + dcompdat From 4977bc52938c058123f2a3f6e0dbf6fc404550dc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 17:36:57 +0200 Subject: [PATCH 0018/1392] implemented direct pack reading - currently not all information is passed on, the absolute offset into the packfile could be interesting to the caller --- fun.py | 4 +-- pack.py | 82 ++++++++++++++++++++++++++++++++++------------- stream.py | 29 ++++++++++++++++- test/test_pack.py | 3 ++ 4 files changed, 92 insertions(+), 26 deletions(-) diff --git a/fun.py b/fun.py index b2e684472..bf223e904 100644 --- a/fun.py +++ b/fun.py @@ -53,7 +53,7 @@ def pack_object_header_info(data): The type_id should be interpreted according to the ``type_id_to_type_map`` map The byte-offset specifies the start of the actual zlib compressed datastream :param m: random-access memory, like a string or memory map""" - c = b0 # first byte + c = ord(data[0]) # first byte i = 1 # next char to read type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size @@ -66,7 +66,7 @@ def pack_object_header_info(data): # END character loop try: - return (type_id_to_type_map[type_id], size) + return (type_id, size, i) except KeyError: # invalid object type - we could try to be smart now and decode part # of the stream to get the info, problem is that we had trouble finding diff --git a/pack.py b/pack.py index 2c01f0452..fa5e43b31 100644 --- a/pack.py +++ b/pack.py @@ -8,6 +8,8 @@ from fun import ( pack_object_header_info, + stream_copy, + chunk_size, OFS_DELTA, REF_DELTA ) @@ -20,6 +22,7 @@ ) from stream import ( DecompressMemMapReader, + NullStream ) from struct import ( @@ -34,50 +37,61 @@ def pack_object_at(data, as_stream): """ - :return: info or stream object of the correct type according to the type - of the object, REF_DELTAS will not be resolved in case a stream is desired. - The resulting ODeltaPackStream will have None instead of a stream. + :return: tuple(num_header_bytes, PackInfo|PackStream) + Tuple of number of additional bytes read from data until the data stream begins + and object of the correct type according to the type of the object. + If as_stream is True, the object will contain a stream, allowing the + data to be read decompressed. :param data: random accessable data at which the header of an object can be read :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only :note: a bit redundant, but it needs to be as fast as possible !""" type_id, uncomp_size, data_offset = pack_object_header_info(data) - + total_offset = None # set later, actual offset until data stream begins + obj = None if type_id == OFS_DELTA: - i = 0 + i = data_offset delta_offset = 0 s = 7 - while c & 0x80: + while True: c = ord(data[i]) - i += 1 delta_offset += (c & 0x7f) << s + i += 1 + if not (c & 0x80): + break s += 7 # END character loop + total_offset = i if as_stream: - stream = DecompressMemMapReader(buffer(data, i), False, uncomp_size) - return ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) + stream = DecompressMemMapReader(buffer(data, total_offset), False, uncomp_size) + obj = ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) else: - return ODeltaPackInfo(type_id, uncomp_size, delta_offset) + obj = ODeltaPackInfo(type_id, uncomp_size, delta_offset) # END handle stream elif type_id == REF_DELTA: - ref_sha = data[:20] + total_offset = data_offset+20 + ref_sha = data[data_offset:total_offset] + if as_stream: - stream = DecompressMemMapReader(buffer(data, 20), False, uncomp_size) - return ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) + stream = DecompressMemMapReader(buffer(data, total_offset), False, uncomp_size) + obj = ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) else: - return ODeltaPackInfo(type_id, uncomp_size, ref_sha) + obj = ODeltaPackInfo(type_id, uncomp_size, ref_sha) # END handle stream else: + total_offset = data_offset # assume its a base object if as_stream: # if no size is given, it will read the header on first access - stream = DecompressMemMapReader(buffer(data, data_offset), False) - return OPackStream(type_id, uncomp_size, stream) + stream = DecompressMemMapReader(buffer(data, data_offset), False, uncomp_size) + obj = OPackStream(type_id, uncomp_size, stream) else: - return OPackInfo(type_id, uncomp_size) + obj = OPackInfo(type_id, uncomp_size) # END handle as_stream # END handle type id + return total_offset, obj + #} END utilities @@ -267,7 +281,8 @@ class PackFile(LazyMixin): __slots__ = ('_packpath', '_data', '_size', '_version') # offset into our data at which the first object starts - _first_object_offset = 3*4 + _first_object_offset = 3*4 # header bytes + _footer_size = 20 # final sha def __init__(self, packpath): self._packpath = packpath @@ -287,16 +302,28 @@ def _set_cache_(self, attr): assert self._version in (2, 3), "Cannot handle pack format version %i" % self._version # END handle header - def _iter_objects(self, start_offset, as_stream): + def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" data = self._data - size = len(data) + content_size = len(data) - self._footer_size cur_offset = start_offset or self._first_object_offset - while cur_offset < size: - ostream = pack_object_at(buffer(data, cur_offset), True) - # TODO: Decompressor needs to track the size of bytes actually decompressed + null = NullStream() + while cur_offset < content_size: + header_offset, ostream = pack_object_at(buffer(data, cur_offset), True) + # scrub the stream to the end - this decompresses the object, but yields + # the amount of compressed bytes we need to get to the next offset + + stream_copy(ostream.read, null.write, ostream.size, chunk_size) + cur_offset += header_offset + ostream.stream.compressed_bytes_read() + + # if a stream is requested, reset it beforehand + # Otherwise return the Stream object directly, its derived from the + # info object + if as_stream: + ostream.stream.seek(0) + yield ostream # END until we have read everything #{ Interface @@ -329,6 +356,15 @@ def stream(self, offset): :return: OPackStream instance, the actual type differs depending on the type_id attribute""" raise NotImplementedError() + def stream_iter(self, start_offset=0): + """:return: iterator yielding OPackStream compatible instances, allowing + to access the data in the pack directly. + :param start_offset: offset to the first object to iterate. If 0, iteration + starts at the very first object in the pack. + :note: Iterating a pack directly is costly as the datastream has to be decompressed + to determine the bounds between the objects""" + return self._iter_objects(start_offset, as_stream=True) + #} END Read-Database like Interface diff --git a/stream.py b/stream.py index fb58125bf..898059def 100644 --- a/stream.py +++ b/stream.py @@ -17,6 +17,21 @@ #{ RO Streams +class NullStream(object): + """A stream that does nothing but providing a stream interface. + Use it like /dev/null""" + __slots__ = tuple() + + def read(self, size=0): + return '' + + def close(self): + pass + + def write(self, data): + return len(data) + + class DecompressMemMapReader(LazyMixin): """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand @@ -144,7 +159,9 @@ def compressed_bytes_read(self): self._br = self._s # END handle stream scrubbing - return self._cbr - len(self._zip.unused_data) + # unused data ends up in the unconsumed tail, which was removed + # from the count already + return self._cbr def seek(self, offset, whence=os.SEEK_SET): """Allows to reset the stream to restart reading @@ -243,7 +260,17 @@ def read(self, size=-1): if dat: dcompdat = dat + dcompdat + # END prepend our cached data + # it can happen, depending on the compression, that we get less bytes + # than ordered as it needs the final portion of the data as well. + # Recursively resolve that. + # Note: dcompdat can be empty even though we still appear to have bytes + # to read, if we are called by compressed_bytes_read - it manipulates + # us to empty the stream + if dcompdat and len(dcompdat) < size and self._br < self._s: + dcompdat += self.read(size-len(dcompdat)) + # END handle special case return dcompdat #} END RO streams diff --git a/test/test_pack.py b/test/test_pack.py index 14d8a2496..a5234a6ce 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -52,6 +52,9 @@ def _assert_pack_file(self, pack, version, size): assert pack.size() == size assert len(pack.checksum()) == 20 + objs = list(pack.stream_iter()) + assert len(objs) == size + def test_pack_index(self): # check version 1 and 2 From ca8236451439490670822103b475df9925884e32 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 16 Jun 2010 21:57:22 +0200 Subject: [PATCH 0019/1392] Implemented offset based pack object collection including test, next up is the actual stream delta handling --- base.py | 59 +++++++++++------ fun.py | 9 +++ pack.py | 157 +++++++++++++++++++++++++++++++--------------- test/test_base.py | 12 +++- test/test_pack.py | 29 ++++++++- 5 files changed, 191 insertions(+), 75 deletions(-) diff --git a/base.py b/base.py index d12181229..aa917858d 100644 --- a/base.py +++ b/base.py @@ -5,7 +5,10 @@ zlib ) -from fun import type_id_to_type_map +from fun import ( + type_id_to_type_map, + type_to_type_id_map + ) __all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', 'OStream', 'OPackStream', 'ODeltaPackStream', @@ -41,6 +44,10 @@ def sha(self): @property def type(self): return self[1] + + @property + def type_id(self): + return type_to_type_id_map[self[1]] @property def size(self): @@ -50,28 +57,40 @@ def size(self): class OPackInfo(tuple): """As OInfo, but provides a type_id property to retrieve the numerical type id, and - does not include a sha""" + does not include a sha. + + Additionally, the pack_offset is the absolute offset into the packfile at which + all object information is located. The data_offset property points to the abosolute + location in the pack at which that actual data stream can be found.""" __slots__ = tuple() - def __new__(cls, type, size): - return tuple.__new__(cls, (type, size)) + def __new__(cls, packoffset, dataoffset, type, size): + return tuple.__new__(cls, (packoffset, dataoffset, type, size)) def __init__(self, *args): tuple.__init__(self) #{ Interface + @property + def pack_offset(self): + return self[0] + + @property + def data_offset(self): + return self[1] + @property def type(self): - return type_id_to_type_map[self[0]] + return type_id_to_type_map[self[2]] @property def type_id(self): - return self[0] + return self[2] @property def size(self): - return self[1] + return self[3] #} END interface @@ -79,17 +98,17 @@ def size(self): class ODeltaPackInfo(OPackInfo): """Adds delta specific information, Either the 20 byte sha which points to some object in the database, - or the base_offset, being an offset into the pack at which our base - can be found""" + or the negative offset from the pack_offset, so that pack_offset - delta_info yields + the pack offset of the base object""" __slots__ = tuple() - def __new__(cls, type, size, delta_info): - return tuple.__new__(cls, (type, size, delta_info)) + def __new__(cls, packoffset, dataoffset, type, size, delta_info): + return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info)) #{ Interface @property def delta_info(self): - return self[2] + return self[4] #} END interface @@ -123,17 +142,17 @@ class OPackStream(OPackInfo): is provided""" __slots__ = tuple() - def __new__(cls, type, size, stream, *args): + def __new__(cls, packoffset, dataoffset, type, size, stream, *args): """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (type, size, stream)) + return tuple.__new__(cls, (packoffset, dataoffset, type, size, stream)) #{ Stream Reader Interface def read(self, size=-1): - return self[2].read(size) + return self[4].read(size) @property def stream(self): - return self[2] + return self[4] #} END stream reader interface @@ -141,17 +160,17 @@ class ODeltaPackStream(ODeltaPackInfo): """Provides a stream outputting the uncompressed offset delta information""" __slots__ = tuple() - def __new__(cls, type, size, delta_info, stream): - return tuple.__new__(cls, (type, size, delta_info, stream)) + def __new__(cls, packoffset, dataoffset, type, size, delta_info, stream): + return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info, stream)) #{ Stream Reader Interface def read(self, size=-1): - return self[3].read(size) + return self[5].read(size) @property def stream(self): - return self[3] + return self[5] #} END stream reader interface diff --git a/fun.py b/fun.py index bf223e904..7999c0a92 100644 --- a/fun.py +++ b/fun.py @@ -24,6 +24,15 @@ REF_DELTA : "REF_DELTA" # REFERENCE DELTA } +type_to_type_id_map = dict( + commit=1, + tree=2, + blob=3, + tag=4, + OFS_DELTA=OFS_DELTA, + REF_DELTA=REF_DELTA + ) + # used when dealing with larger streams chunk_size = 1000*1000 diff --git a/pack.py b/pack.py index fa5e43b31..8de2a47db 100644 --- a/pack.py +++ b/pack.py @@ -1,4 +1,7 @@ """Contains PackIndexFile and PackFile implementations""" +from gitdb.exc import ( + BadObject, + ) from util import ( LockedFD, LazyMixin, @@ -31,67 +34,68 @@ __all__ = ('PackIndexFile', 'PackFile') +_delta_types = (OFS_DELTA, REF_DELTA) #{ Utilities -def pack_object_at(data, as_stream): +def pack_object_at(data, offset, as_stream): """ - :return: tuple(num_header_bytes, PackInfo|PackStream) - Tuple of number of additional bytes read from data until the data stream begins - and object of the correct type according to the type of the object. + :return: PackInfo|PackStream + an object of the correct type according to the type_id of the object. If as_stream is True, the object will contain a stream, allowing the data to be read decompressed. - :param data: random accessable data at which the header of an object can be read + :param data: random accessable data containing all required information + :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read - the data, otherwise you receive an info object only - :note: a bit redundant, but it needs to be as fast as possible !""" - type_id, uncomp_size, data_offset = pack_object_header_info(data) - total_offset = None # set later, actual offset until data stream begins - obj = None + the data, otherwise you receive an info object only""" + ldata = len(data) # debug + data = buffer(data, offset) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) + total_rela_offset = None # set later, actual offset until data stream begins + delta_info = None + + # OFFSET DELTA if type_id == OFS_DELTA: - i = data_offset - delta_offset = 0 - s = 7 - while True: + i = data_rela_offset + c = ord(data[i]) + i += 1 + delta_offset = c & 0x7f + while c & 0x80: c = ord(data[i]) - delta_offset += (c & 0x7f) << s i += 1 - if not (c & 0x80): - break - s += 7 + delta_offset += 1 + delta_offset = (delta_offset << 7) + (c & 0x7f) # END character loop - total_offset = i - if as_stream: - stream = DecompressMemMapReader(buffer(data, total_offset), False, uncomp_size) - obj = ODeltaPackStream(type_id, uncomp_size, delta_offset, stream) - else: - obj = ODeltaPackInfo(type_id, uncomp_size, delta_offset) - # END handle stream + delta_info = delta_offset + total_rela_offset = i + # REF DELTA elif type_id == REF_DELTA: - total_offset = data_offset+20 - ref_sha = data[data_offset:total_offset] - - if as_stream: - stream = DecompressMemMapReader(buffer(data, total_offset), False, uncomp_size) - obj = ODeltaPackStream(type_id, uncomp_size, ref_sha, stream) - else: - obj = ODeltaPackInfo(type_id, uncomp_size, ref_sha) - # END handle stream + total_rela_offset = data_rela_offset+20 + ref_sha = data[data_rela_offset:total_rela_offset] + delta_info = ref_sha + # BASE OBJECT else: - total_offset = data_offset # assume its a base object - if as_stream: - # if no size is given, it will read the header on first access - stream = DecompressMemMapReader(buffer(data, data_offset), False, uncomp_size) - obj = OPackStream(type_id, uncomp_size, stream) - else: - obj = OPackInfo(type_id, uncomp_size) - # END handle as_stream + total_rela_offset = data_rela_offset # END handle type id - return total_offset, obj - + abs_data_offset = offset + total_rela_offset + if as_stream: + stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) + if delta_info is None: + return OPackStream(offset, abs_data_offset, type_id, uncomp_size, stream) + else: + return ODeltaPackStream(offset, abs_data_offset, type_id, uncomp_size, delta_info, stream) + else: + if delta_info is None: + return OPackInfo(offset, abs_data_offset, type_id, uncomp_size) + else: + return ODeltaPackInfo(offset, abs_data_offset, type_id, uncomp_size, delta_info) + # END handle info + # END handle stream + + #} END utilities @@ -310,12 +314,12 @@ def _iter_objects(self, start_offset, as_stream=True): null = NullStream() while cur_offset < content_size: - header_offset, ostream = pack_object_at(buffer(data, cur_offset), True) + ostream = pack_object_at(data, cur_offset, True) # scrub the stream to the end - this decompresses the object, but yields # the amount of compressed bytes we need to get to the next offset stream_copy(ostream.read, null.write, ostream.size, chunk_size) - cur_offset += header_offset + ostream.stream.compressed_bytes_read() + cur_offset += (ostream.data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() # if a stream is requested, reset it beforehand @@ -326,7 +330,7 @@ def _iter_objects(self, start_offset, as_stream=True): yield ostream # END until we have read everything - #{ Interface + #{ Pack Information def size(self): """:return: The amount of objects stored in this pack""" @@ -340,7 +344,58 @@ def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" return self._data[-20:] - #} END interface + #} END pack information + + #{ Pack Specific + + def collect_streams(self, offset): + """ + :return: list of pack streams which are required to build the object + at the given offset. The first entry of the list is the object at offset, + the last one is either a full object, or a REF_Delta stream. The latter + type needs its reference object to be locked up in an ODB to form a valid + delta chain. + :param offset: specifies the first byte of the object within this pack""" + out = list() + while True: + ostream = pack_object_at(self._data, offset, True) + out.append(ostream) + if ostream.type_id == OFS_DELTA: + offset = ostream.pack_offset - ostream.delta_info + else: + # the only thing we can lookup are OFFSET deltas. Everything + # else is either an object, or a ref delta, in the latter + # case someone else has to find it + break + # END handle type + # END while chaining streams + return out + + def to_delta_stream(self, stream_list): + """Convert the given list of streams into a stream which resolves deltas + (if availble) when reading from it. + :param stream_list: one or more stream objects. If the first stream is a Delta, + there must be at least two streams in the list. The list's last stream + must be a non-delta stream. + :return: Non-Delta OPackStream object whose stream can be used to obtain + the decompressed resolved data + :raise ValueError: if the stream list cannot be handled due to a missing base object""" + if len(stream_list) == 1: + if stream_list[0].type_id in _delta_types: + raise ValueError("Cannot resolve deltas if only one stream is given", stream_list[0].type) + # its an object, no need to resolve anything + return stream_list[0] + # END single object special handling + + if stream_list[-1].type_id in _delta_types: + raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) + # END check stream + + # just create the respective stream wrapper + raise NotImplementedError() + + + #} END pack specific #{ Read-Database like Interface @@ -348,13 +403,13 @@ def info(self, offset): """Retrieve information about the object at the given file-absolute offset :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - raise NotImplementedError() + return pack_object_at(self._data, offset or self._first_object_offset, False) def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - raise NotImplementedError() + return pack_object_at(self._data, offset or self._first_object_offset, True) def stream_iter(self, start_offset=0): """:return: iterator yielding OPackStream compatible instances, allowing @@ -390,12 +445,14 @@ def _iter_objects(self, as_stream): def info(self, sha): """Retrieve information about the object identified by the given sha :param sha: 20 byte sha1 + :raise BadObject: :return: OInfo instance""" raise NotImplementedError() def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha :param sha: 20 byte sha1 + :raise BadObject: :return: OStream instance""" raise NotImplementedError() diff --git a/test/test_base.py b/test/test_base.py index f7ddebaa2..524bf3054 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -21,22 +21,28 @@ def test_streams(self): # test info sha = NULL_HEX_SHA s = 20 + blob_id = 3 + info = OInfo(sha, str_blob_type, s) assert info.sha == sha assert info.type == str_blob_type + assert info.type_id == blob_id assert info.size == s # test pack info # provides type_id - blob_id = 3 - pinfo = OPackInfo(blob_id, s) + pinfo = OPackInfo(0, 1, blob_id, s) assert pinfo.type == str_blob_type assert pinfo.type_id == blob_id + assert pinfo.pack_offset == 0 + assert pinfo.data_offset == 1 - dpinfo = ODeltaPackInfo(blob_id, s, sha) + dpinfo = ODeltaPackInfo(0, 1, blob_id, s, sha) assert dpinfo.type == str_blob_type assert dpinfo.type_id == blob_id assert dpinfo.delta_info == sha + assert dpinfo.pack_offset == 0 + assert dpinfo.data_offset == 1 # test ostream diff --git a/test/test_pack.py b/test/test_pack.py index a5234a6ce..f2b0f9481 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -52,8 +52,33 @@ def _assert_pack_file(self, pack, version, size): assert pack.size() == size assert len(pack.checksum()) == 20 - objs = list(pack.stream_iter()) - assert len(objs) == size + num_obj = 0 + for obj in pack.stream_iter(): + num_obj += 1 + info = pack.info(obj.pack_offset) + stream = pack.stream(obj.pack_offset) + + assert info.pack_offset == stream.pack_offset + assert info.data_offset == stream.data_offset + assert info.type_id == stream.type_id + assert hasattr(stream, 'read') + + # it should be possible to read from both streams + assert obj.read() == stream.read() + + streams = pack.collect_streams(obj.pack_offset) + assert streams + + # read the stream + try: + dstream = pack.to_delta_stream(streams) + except ValueError: + # ignore these, old git versions use only ref deltas, + # which we havent resolved ( as we are without an index ) + continue + # END get deltastream + # END for each object + assert num_obj == size def test_pack_index(self): From 84c4e5a33aac3010d197a3a92adbff52a83969da Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 00:39:17 +0200 Subject: [PATCH 0020/1392] initial research on possible delta-apply algorithms. True streaming appears only possible if delta opcodes are acessing only sequential memory, but through mmaps, it should still be possible to obtain decent performance even on big files --- pack.py | 6 ++-- stream.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++ test/test_pack.py | 8 +++++ 3 files changed, 86 insertions(+), 3 deletions(-) diff --git a/pack.py b/pack.py index 8de2a47db..314ae07c1 100644 --- a/pack.py +++ b/pack.py @@ -25,7 +25,8 @@ ) from stream import ( DecompressMemMapReader, - NullStream + DeltaApplyReader, + NullStream, ) from struct import ( @@ -49,7 +50,6 @@ def pack_object_at(data, offset, as_stream): :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" - ldata = len(data) # debug data = buffer(data, offset) type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins @@ -392,7 +392,7 @@ def to_delta_stream(self, stream_list): # END check stream # just create the respective stream wrapper - raise NotImplementedError() + return DeltaApplyReader(stream_list) #} END pack specific diff --git a/stream.py b/stream.py index 898059def..735924845 100644 --- a/stream.py +++ b/stream.py @@ -272,7 +272,82 @@ def read(self, size=-1): dcompdat += self.read(size-len(dcompdat)) # END handle special case return dcompdat + + +class DeltaApplyReader(LazyMixin): + """A reader which dynamically applies pack deltas to a base object, keeping the + memory demands to a minimum. + + The size of the final object is only obtainable once all deltas have been + applied, unless it is retrieved from a pack index. + + The uncompressed Delta has the following layout (MSB being a most significant + bit encoded dynamic size): + + * MSB Source Size - the size of the base against which the delta was created + * MSB Target Size - the size of the resulting data after the delta was applied + * A list of one byte commands (cmd) which are followed by a specific protocol: + + * cmd & 0x80 - copy delta_data[offset:offset+size] + + * Followed by an encoded offset into the delta data + * Followed by an encoded size of the chunk to copy + + * cmd & 0x7f - insert + + * insert cmd bytes from the delta buffer into the output stream + + * cmd == 0 - invalid operation ( or error in delta stream ) + """ + __slots__ = ( + "_streams", # tuple of our stream objects + "_readers", # list of read methods from our streams + "_mm_target", # memory map of the delta-applied data + ) + + def __init__(self, stream_list): + """Initialize this instance with a list of streams, the first stream being + the delta to apply on top of all following deltas, the last stream being the + base object onto which to apply the deltas""" + assert len(stream_list) > 1, "Need at least one delta and one base stream" + + self._streams = tuple(stream_list) + self._readers = None # TODO + + def _set_cache_(self, attr): + """If we are here, we apply the actual deltas""" + # fill in delta info structures, providing the source and target buffer + # sizes. + # Allocate private memory map big enough to hold the first base buffer + # It can be swapped out if it is too large. We need random access to it + + # allocate memory map large enough for the largest (intermediate) target + # We will use it as scratch space for all delta ops. If the final + # target buffer is smaller than our allocated space, we just use parts + # of it + + # for each delta to apply, memory map the decompressed delta and + # work on the op-codes to reconstruct everything. + # For the actual copying, we use a seek and write pattern of buffer + # slices. + + # NOTE: on py pre 2.5, all memory maps must actually be some kind + # of memory buffer,like StringIO ( ouch ;) ) + + + + # TODO: Once that works, figure out the ordering of the opcodes. If they + # are always in-order/sequential, an alternate implementation could + # use stream access only. Of course this would mean we would read + # all deltas in advance, analyse the opcode ranges to determine a final + # concatenated opcode list which indicates what to copy from which delta + # to which position. This preprocessing would allow true streaming + + def read(self, size=0): + # pass the call to our lazy-loaded delta-applied data + return self._mm_target.read(size) + #} END RO streams diff --git a/test/test_pack.py b/test/test_pack.py index f2b0f9481..d972bef14 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -77,6 +77,14 @@ def _assert_pack_file(self, pack, version, size): # which we havent resolved ( as we are without an index ) continue # END get deltastream + + # TODO: TestStream._assert_stream_reader does that already, should + # be used instead + # read all + dstream.read() + + # read chunks + # END for each object assert num_obj == size From 6a4eee20486eca91d06d7ba1420c8a31bcf0f4a8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 11:20:06 +0200 Subject: [PATCH 0021/1392] initial version of delta-apply, but more pedandic testing is required --- fun.py | 89 ++++++++++++++++++++++++++++++++++++++++- stream.py | 100 +++++++++++++++++++++++++++++++++++++++------- test/test_pack.py | 6 +-- util.py | 13 ++++++ 4 files changed, 188 insertions(+), 20 deletions(-) diff --git a/fun.py b/fun.py index 7999c0a92..52688ba7b 100644 --- a/fun.py +++ b/fun.py @@ -9,6 +9,7 @@ from util import zlib decompressobj = zlib.decompressobj +import mmap # INVARIANTS OFS_DELTA = 6 @@ -34,7 +35,7 @@ ) # used when dealing with larger streams -chunk_size = 1000*1000 +chunk_size = 1000*mmap.PAGESIZE __all__ = ('is_loose_object', 'loose_object_header_info', 'object_header_info', 'write_object' ) @@ -83,6 +84,26 @@ def pack_object_header_info(data): raise BadObjectType(type_id) # END handle exceptions +def msb_size(data, offset=0): + """:return: tuple(read_bytes, size) read the msb size from the given random + access data starting at the given byte offset""" + size = 0 + i = 0 + l = len(data) + hit_msb = False + while i < l: + c = ord(data[i+offset]) + size |= (c & 0x7f) << i*7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + if not hit_msb: + raise AssertionError("Could not find terminating MSB byte in data stream") + return i+offset, size + def write_object(type, size, read, write, chunk_size=chunk_size): """Write the object as identified by type, size and source_stream into the target_stream @@ -111,8 +132,15 @@ def stream_copy(read, write, size, chunk_size): # WRITE ALL DATA UP TO SIZE while True: cs = min(chunk_size, size-dbw) - data_len = write(read(cs)) + # NOTE: not all write methods return the amount of written bytes, like + # mmap.write. Its bad, but we just deal with it ... perhaps its not + # even less efficient + # data_len = write(read(cs)) + # dbw += data_len + data = read(cs) + data_len = len(data) dbw += data_len + write(data) if data_len < cs or dbw == size: break # END check for stream end @@ -120,5 +148,62 @@ def stream_copy(read, write, size, chunk_size): return dbw +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): + """Apply data from a delta buffer using a source buffer to the target file, + which will be written to + :param src_buf: random access data from which the delta was created + :param src_buf_size: size of the source buffer in bytes + :param delta_buf_size: size fo the delta buffer in bytes + :param delta_buf: random access delta data + :param target_file: file like object to write the result to + :note: transcribed to python from the similar routine in patch-delta.c""" + i = 0 + twrite = target_file.write + db = delta_buf + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += i + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + # maybe skip this check ? + if (cp_off + cp_size < cp_size or + cp_off + cp_size > src_buf_size): + break + twrite(src_buf[cp_off:cp_off+cp_size]) + elif c: + twrite(db[i:i+c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + # yes, lets use the exact same error message that git uses :) + assert i == delta_buf_size, "delta replay has gone wild" + #} END routines diff --git a/stream.py b/stream.py index 735924845..5ced7ada0 100644 --- a/stream.py +++ b/stream.py @@ -4,7 +4,14 @@ import mmap import os +from fun import ( + msb_size, + stream_copy, + apply_delta_data + ) + from util import ( + allocate_memory, LazyMixin, make_sha, write, @@ -300,9 +307,11 @@ class DeltaApplyReader(LazyMixin): * cmd == 0 - invalid operation ( or error in delta stream ) """ __slots__ = ( - "_streams", # tuple of our stream objects - "_readers", # list of read methods from our streams + "_bstream", # base stream to which to apply the deltas + "_dstreams", # tuple of delta stream readers "_mm_target", # memory map of the delta-applied data + "_size", # actual number of bytes in _mm_target + "_br" # number of bytes read ) def __init__(self, stream_list): @@ -311,31 +320,81 @@ def __init__(self, stream_list): base object onto which to apply the deltas""" assert len(stream_list) > 1, "Need at least one delta and one base stream" - self._streams = tuple(stream_list) - self._readers = None # TODO + self._bstream = stream_list[-1] + self._dstreams = tuple(stream_list[:-1]) + self._br = 0 def _set_cache_(self, attr): """If we are here, we apply the actual deltas""" # fill in delta info structures, providing the source and target buffer # sizes. + buffer_offset_list = list() + final_target_size = None + max_target_size = 0 + for dstream in self._dstreams: + buf = dstream.read(512) # read the header information + X + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + if final_target_size is None: + final_target_size = target_size + # END set final target size + buffer_offset_list.append((buffer(buf, offset), offset)) + max_target_size = max(max_target_size, target_size) + # END for each delta stream + + # sanity check - the first delta to apply should have the same source + # size as our actual base stream + base_size = self._bstream.size + target_size = max_target_size + + # if we have more than 1 delta to apply, we will swap buffers, hence we must + # assure that all buffers we use are large enough to hold all the results + if len(self._dstreams) > 1: + base_size = target_size = max(base_size, max_target_size) + # END adjust buffer sizes + # Allocate private memory map big enough to hold the first base buffer - # It can be swapped out if it is too large. We need random access to it + # We need random access to it + bbuf = allocate_memory(base_size) # allocate memory map large enough for the largest (intermediate) target # We will use it as scratch space for all delta ops. If the final # target buffer is smaller than our allocated space, we just use parts - # of it + # of it upon return. + tbuf = allocate_memory(target_size) # for each delta to apply, memory map the decompressed delta and # work on the op-codes to reconstruct everything. # For the actual copying, we use a seek and write pattern of buffer # slices. - - # NOTE: on py pre 2.5, all memory maps must actually be some kind - # of memory buffer,like StringIO ( ouch ;) ) - - + for (dbuf, offset), dstream in reversed(zip(buffer_offset_list, self._dstreams)): + # allocate a buffer to hold all delta data - fill in the data for + # fast access. We do this as we know that reading individual bytes + # from our stream would be slower than necessary ( although possible ) + # The dbuf buffer contains commands after the first two MSB sizes, the + # offset specifies the amount of bytes read to get the sizes. + ddata = allocate_memory(dstream.size - offset) + ddata.write(dbuf) + # read the rest from the stream. The size we give is larger than necessary + stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) + + ################################################################ + apply_delta_data(bbuf, len(bbuf), ddata, len(ddata), tbuf) + ################################################################ + + # finally, swap out source and target buffers. The target is now the + # base for the next delta to apply + bbuf, tbuf = tbuf, bbuf + bbuf.seek(0) + tbuf.seek(0) + # END for each delta to apply + + # its already seeked to 0, constrain it to the actual size + # NOTE: in the end of the loop, it swaps buffers, hence our target buffer + # is not tbuf, but bbuf ! + self._mm_target = bbuf + self._size = final_target_size # TODO: Once that works, figure out the ordering of the opcodes. If they # are always in-order/sequential, an alternate implementation could @@ -344,10 +403,21 @@ def _set_cache_(self, attr): # concatenated opcode list which indicates what to copy from which delta # to which position. This preprocessing would allow true streaming - def read(self, size=0): - # pass the call to our lazy-loaded delta-applied data - return self._mm_target.read(size) - + def read(self, count=0): + bl = self._size - self._br # bytes left + if count < 1 or count > bl: + count = bl + data = self._mm_target.read(count) + self._br += len(data) + return data + + def seek(self, offset, whence=os.SEEK_SET): + """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != os.SEEK_SET: + raise ValueError("Can only seek to position 0") + # END handle offset + self._size #} END RO streams diff --git a/test/test_pack.py b/test/test_pack.py index d972bef14..5786fbf72 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -78,12 +78,12 @@ def _assert_pack_file(self, pack, version, size): continue # END get deltastream - # TODO: TestStream._assert_stream_reader does that already, should - # be used instead # read all - dstream.read() + assert len(dstream.read()) # read chunks + # NOTE: the current implementation is safe, it basically transfers + # all calls to the underlying memory map # END for each object assert num_obj == size diff --git a/util.py b/util.py index 5c2bb540b..f10b71b12 100644 --- a/util.py +++ b/util.py @@ -94,6 +94,19 @@ def stream_copy(source, destination, chunk_size=512*1024): # END reading output stream return br +def allocate_memory(size): + """:return: a file-protocol accessible memory block of the given size""" + try: + return mmap.mmap(-1, size) # read-write by default + except EnvironmentError: + # setup real memory instead + # this of course may fail if the amount of memory is not available in + # one chunk - would only be the case in python 2.4, being more likely on + # 32 bit systems. + return cStringIO.StringIO("\0"*size) + # END handle memory allocation + + def file_contents_ro(fd, stream=False, allow_mmap=True): """:return: read-only contents of the file represented by the file descriptor fd :param fd: file descriptor opened for reading From f4b6e272963fefdb1e372b87a09e7c74680d6b52 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 13:24:41 +0200 Subject: [PATCH 0022/1392] Implemented main PackEntity object retrieval method and moved constructor for delta_streams out of the PackFile, into the stream itself where it belongs. All this is still to be tested --- fun.py | 2 + pack.py | 145 ++++++++++++++++++++++++++++++++++------------ stream.py | 49 +++++++++++++++- test/test_pack.py | 9 ++- 4 files changed, 165 insertions(+), 40 deletions(-) diff --git a/fun.py b/fun.py index 52688ba7b..cbc37b2f0 100644 --- a/fun.py +++ b/fun.py @@ -14,6 +14,8 @@ # INVARIANTS OFS_DELTA = 6 REF_DELTA = 7 +delta_types = (OFS_DELTA, REF_DELTA) + type_id_to_type_map = { 0 : "", # EXT 1 1 : "commit", diff --git a/pack.py b/pack.py index 314ae07c1..811acaf98 100644 --- a/pack.py +++ b/pack.py @@ -5,19 +5,24 @@ from util import ( LockedFD, LazyMixin, - file_contents_ro, - unpack_from + unpack_from, + file_contents_ro, ) from fun import ( pack_object_header_info, + type_id_to_type_map, stream_copy, chunk_size, + delta_types, OFS_DELTA, - REF_DELTA + REF_DELTA, + msb_size ) -from base import ( +from base import ( # Amazing ! + OInfo, + OStream, OPackInfo, OPackStream, ODeltaPackInfo, @@ -35,7 +40,7 @@ __all__ = ('PackIndexFile', 'PackFile') -_delta_types = (OFS_DELTA, REF_DELTA) + #{ Utilities @@ -95,8 +100,6 @@ def pack_object_at(data, offset, as_stream): # END handle info # END handle stream - - #} END utilities @@ -355,6 +358,7 @@ def collect_streams(self, offset): the last one is either a full object, or a REF_Delta stream. The latter type needs its reference object to be locked up in an ODB to form a valid delta chain. + If the object at offset is no delta, the size of the list is 1. :param offset: specifies the first byte of the object within this pack""" out = list() while True: @@ -370,31 +374,7 @@ def collect_streams(self, offset): # END handle type # END while chaining streams return out - - def to_delta_stream(self, stream_list): - """Convert the given list of streams into a stream which resolves deltas - (if availble) when reading from it. - :param stream_list: one or more stream objects. If the first stream is a Delta, - there must be at least two streams in the list. The list's last stream - must be a non-delta stream. - :return: Non-Delta OPackStream object whose stream can be used to obtain - the decompressed resolved data - :raise ValueError: if the stream list cannot be handled due to a missing base object""" - if len(stream_list) == 1: - if stream_list[0].type_id in _delta_types: - raise ValueError("Cannot resolve deltas if only one stream is given", stream_list[0].type) - # its an object, no need to resolve anything - return stream_list[0] - # END single object special handling - - if stream_list[-1].type_id in _delta_types: - raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) - # END check stream - - # just create the respective stream wrapper - return DeltaApplyReader(stream_list) - - + #} END pack specific #{ Read-Database like Interface @@ -437,8 +417,58 @@ def __init__(self, basename): self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + def _sha_to_index(self, sha): + """:return: index for the given sha, or raise""" + index = self._index.sha_to_index(sha) + if index is None: + raise BadObject(sha) + return index + def _iter_objects(self, as_stream): - raise NotImplementedError + """Iterate over all objects in our index and yield their OInfo or OStream instences""" + raise NotImplementedError() + + def _object(self, sha, as_stream): + """:return: OInfo or OStream object providing information about the given sha""" + # its a little bit redundant here, but it needs to be efficient + offset = self._index.offset(self._sha_to_index(sha)) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset)) + if as_stream: + if type_id not in delta_types: + packstream = self._pack.stream(offset) + return OStream(sha, packstream.type, packstream.size, packstream.stream) + # END handle non-deltas + + # produce a delta stream containing all info + # To prevent it from applying the deltas when querying the size, + # we extract it from the delta stream ourselves + streams = self.collect_streams_at_offset(offset) + buf = streams[0].read(512) + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + + streams[0].seek(0) # assure it can be read by the delta reader + dstream = DeltaApplyReader.new(streams) + + return OStream(sha, dstream.type, target_size, dstream) + else: + if type_id not in delta_types: + return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) + # END handle non-deltas + + # deltas are a little tougher - unpack the first bytes to obtain + # the actual target size, as opposed to the size of the delta data + streams = self.collect_streams_at_offset(offset) + buf = streams[0].read(512) + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + + # collect the streams to obtain the actual object type + if streams[-1].type_id in delta_types: + raise BadObject(sha, "Could not resolve delta object") + + return OInfo(sha, streams[-1].type, target_size) + # END handle stream #{ Read-Database like Interface @@ -447,14 +477,14 @@ def info(self, sha): :param sha: 20 byte sha1 :raise BadObject: :return: OInfo instance""" - raise NotImplementedError() + return self._object(sha, as_stream=False) def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha :param sha: 20 byte sha1 :raise BadObject: :return: OStream instance""" - raise NotImplementedError() + return self._object(sha, as_stream=True) #} END Read-Database like Interface @@ -470,4 +500,47 @@ def stream_iter(self): OStream instances""" return self._iter_objects(as_stream=True) - #} Interface + def collect_streams_at_offset(self, offset): + """As the version in the PackFile, but can resolve REF deltas within this pack + For more info, see ``collect_streams`` + :param offset: offset into the pack file at which the object can be found""" + streams = self._pack.collect_streams(offset) + + # try to resolve the last one if needed. It is assumed to be either + # a REF delta, or a base object, as OFFSET deltas are resolved by the pack + if streams[-1].type_id == REF_DELTA: + stream = streams[-1] + while stream.type_id in delta_types: + if stream.type_id == REF_DELTA: + sindex = self._index.sha_to_index(stream.delta_info) + if sindex is None: + break + stream = self._pack.stream(self._index.offset(sindex)) + streams.append(stream) + else: + # must be another OFS DELTA - this could happen if a REF + # delta we resolve previously points to an OFS delta. Who + # would do that ;) ? We can handle it though + stream = self._pack.stream(stream.delta_info) + streams.append(stream) + # END handle ref delta + # END resolve ref streams + # END resolve streams + + return streams + + def collect_streams(self, sha): + """As ``PackFile.collect_streams``, but takes a sha instead of an offset. + Additionally, ref_delta streams will be resolved within this pack. + If this is not possible, the stream will be left alone, hence it is adivsed + to check for unresolved ref-deltas and resolve them before attempting to + construct a delta stream. + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect + :return: list of streams, first being the actual object delta, the last being + a possibly unresolved base object. + :raise BadObject:""" + return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) + + + + #} END interface diff --git a/stream.py b/stream.py index 5ced7ada0..10a8e057c 100644 --- a/stream.py +++ b/stream.py @@ -7,7 +7,8 @@ from fun import ( msb_size, stream_copy, - apply_delta_data + apply_delta_data, + delta_types ) from util import ( @@ -19,7 +20,7 @@ zlib ) -__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer') +__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader') #{ RO Streams @@ -418,6 +419,50 @@ def seek(self, offset, whence=os.SEEK_SET): raise ValueError("Can only seek to position 0") # END handle offset self._size + + #{ Interface + + @classmethod + def new(cls, stream_list): + """Convert the given list of streams into a stream which resolves deltas + when reading from it. + :param stream_list: two or more stream objects, first stream is a Delta + to the object that you want to resolve, followed by N additional delta + streams. The list's last stream must be a non-delta stream. + :return: Non-Delta OPackStream object whose stream can be used to obtain + the decompressed resolved data + :raise ValueError: if the stream list cannot be handled""" + if len(stream_list) < 2: + raise ValueError("Need at least two streams") + # END single object special handling + + if stream_list[-1].type_id in delta_types: + raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) + # END check stream + + return cls(stream_list) + + #} END interface + + + #{ OInfo like Interface + + @property + def type(self): + return self._bstream.type + + @property + def type_id(self): + return self._bstream.type_id + + @property + def size(self): + """:return: number of uncompressed bytes in the stream""" + return self._size + + #} END oinfo like interface + + #} END RO streams diff --git a/test/test_pack.py b/test/test_pack.py index 5786fbf72..1f860961e 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -5,6 +5,10 @@ with_packs_rw, fixture_path ) +from gitdb.stream import ( + DeltaApplyReader + ) + from gitdb.pack import ( PackIndexFile, PackFile @@ -71,15 +75,16 @@ def _assert_pack_file(self, pack, version, size): # read the stream try: - dstream = pack.to_delta_stream(streams) + dstream = DeltaApplyReader.new(streams) except ValueError: # ignore these, old git versions use only ref deltas, # which we havent resolved ( as we are without an index ) + # Also ignore non-delta streams continue # END get deltastream # read all - assert len(dstream.read()) + assert len(dstream.read()) # read chunks # NOTE: the current implementation is safe, it basically transfers From 001f030cc8c87407d30fe87fe24929cc1edb8f48 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 16:51:39 +0200 Subject: [PATCH 0023/1392] Initial implementation of stream validation - this is the final hurdle, if that works ( which it doesn't for yet for everything ), than the pack reading would officially work --- fun.py | 7 ++- pack.py | 124 ++++++++++++++++++++++++++++++++++++++-------- stream.py | 2 +- test/test_pack.py | 38 ++++++++++++-- util.py | 15 ------ 5 files changed, 146 insertions(+), 40 deletions(-) diff --git a/fun.py b/fun.py index cbc37b2f0..1466a78bb 100644 --- a/fun.py +++ b/fun.py @@ -106,6 +106,11 @@ def msb_size(data, offset=0): raise AssertionError("Could not find terminating MSB byte in data stream") return i+offset, size +def loose_object_header(type, size): + """:return: string representing the loose object header, which is immediately + followed by the content stream of size 'size'""" + return "%s %i\0" % (type, size) + def write_object(type, size, read, write, chunk_size=chunk_size): """Write the object as identified by type, size and source_stream into the target_stream @@ -120,7 +125,7 @@ def write_object(type, size, read, write, chunk_size=chunk_size): tbw = 0 # total num bytes written # WRITE HEADER: type SP size NULL - tbw += write("%s %i\0" % (type, size)) + tbw += write(loose_object_header(type, size)) tbw += stream_copy(read, write, size, chunk_size) return tbw diff --git a/pack.py b/pack.py index 811acaf98..b1f2f1e05 100644 --- a/pack.py +++ b/pack.py @@ -3,6 +3,7 @@ BadObject, ) from util import ( + zlib, LockedFD, LazyMixin, unpack_from, @@ -12,6 +13,7 @@ from fun import ( pack_object_header_info, type_id_to_type_map, + write_object, stream_copy, chunk_size, delta_types, @@ -31,6 +33,7 @@ from stream import ( DecompressMemMapReader, DeltaApplyReader, + Sha1Writer, NullStream, ) @@ -38,7 +41,8 @@ pack, ) -__all__ = ('PackIndexFile', 'PackFile') +import os +__all__ = ('PackIndexFile', 'PackFile', 'PackEntity') @@ -237,6 +241,10 @@ def size(self): """:return: amount of objects referred to by this index""" return self._fanout_table[255] + def path(self): + """:return: path to the packindexfile""" + return self._indexpath + def packfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of the pack file""" return self._data[-40:-20] @@ -288,8 +296,8 @@ class PackFile(LazyMixin): __slots__ = ('_packpath', '_data', '_size', '_version') # offset into our data at which the first object starts - _first_object_offset = 3*4 # header bytes - _footer_size = 20 # final sha + first_object_offset = 3*4 # header bytes + footer_size = 20 # final sha def __init__(self, packpath): self._packpath = packpath @@ -312,8 +320,8 @@ def _set_cache_(self, attr): def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" data = self._data - content_size = len(data) - self._footer_size - cur_offset = start_offset or self._first_object_offset + content_size = len(data) - self.footer_size + cur_offset = start_offset or self.first_object_offset null = NullStream() while cur_offset < content_size: @@ -343,10 +351,18 @@ def version(self): """:return: the version of this pack""" return self._version + def data(self): + """:return: read-only data of this pack. It provides random access and usually + is a memory map""" + return self._data + def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" return self._data[-20:] - + + def path(self): + """:return: path to the packfile""" + return self._packpath #} END pack information #{ Pack Specific @@ -383,13 +399,13 @@ def info(self, offset): """Retrieve information about the object at the given file-absolute offset :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self._first_object_offset, False) + return pack_object_at(self._data, offset or self.first_object_offset, False) def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self._first_object_offset, True) + return pack_object_at(self._data, offset or self.first_object_offset, True) def stream_iter(self, start_offset=0): """:return: iterator yielding OPackStream compatible instances, allowing @@ -403,7 +419,7 @@ def stream_iter(self, start_offset=0): #} END Read-Database like Interface -class PackFileEntity(object): +class PackEntity(object): """Combines the PackIndexFile and the PackFile into one, allowing the actual objects to be resolved and iterated""" @@ -412,11 +428,12 @@ class PackFileEntity(object): IndexFileCls = PackIndexFile PackFileCls = PackFile - def __init__(self, basename): + def __init__(self, pack_or_index_path): + """Initialize ourselves with the path to the respective pack or index file""" + basename, ext = os.path.splitext(pack_or_index_path) self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance - def _sha_to_index(self, sha): """:return: index for the given sha, or raise""" index = self._index.sha_to_index(sha) @@ -426,12 +443,20 @@ def _sha_to_index(self, sha): def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" - raise NotImplementedError() - - def _object(self, sha, as_stream): - """:return: OInfo or OStream object providing information about the given sha""" + indexfile = self._index + _object = self._object + for index in xrange(indexfile.size()): + sha = indexfile.sha(index) + yield _object(sha, as_stream, index) + # END for each index + + def _object(self, sha, as_stream, index=-1): + """:return: OInfo or OStream object providing information about the given sha + :param index: if not -1, its assumed to be the sha's index in the IndexFile""" # its a little bit redundant here, but it needs to be efficient - offset = self._index.offset(self._sha_to_index(sha)) + if index < 0: + index = self._sha_to_index(sha) + offset = self._index.offset(index) type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset)) if as_stream: if type_id not in delta_types: @@ -447,7 +472,7 @@ def _object(self, sha, as_stream): offset, src_size = msb_size(buf) offset, target_size = msb_size(buf, offset) - streams[0].seek(0) # assure it can be read by the delta reader + streams[0].stream.seek(0) # assure it can be read by the delta reader dstream = DeltaApplyReader.new(streams) return OStream(sha, dstream.type, target_size, dstream) @@ -476,20 +501,79 @@ def info(self, sha): """Retrieve information about the object identified by the given sha :param sha: 20 byte sha1 :raise BadObject: - :return: OInfo instance""" + :return: OInfo instance, with 20 byte sha""" return self._object(sha, as_stream=False) def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha :param sha: 20 byte sha1 :raise BadObject: - :return: OStream instance""" + :return: OStream instance, with 20 byte sha""" return self._object(sha, as_stream=True) #} END Read-Database like Interface #{ Interface - + + def pack(self): + """:return: the underlying pack file instance""" + return self._pack + + def index(self): + """:return: the underlying pack index file instance""" + return self._index + + def is_valid_stream(self, sha, use_crc=False): + """Verify that the stream at the given sha is valid. + :param sha: 20 byte sha1 of the object whose stream to verify + :param use_crc: if True, the index' crc for the sha is used to determine + whether the compressed stream of the object is valid. If it is + a delta, this only verifies that the delta's data is valid, not the + data of the actual undeltified object, as it depends on more than + just this stream. + If False, the object will be decompressed and the sha generated. It must + match the given sha + :return: True if the stream is valid + :raise UnsupportedOperation: If the index is version 1 only + :raise BadObject: sha was not found""" + if use_crc: + index = self._sha_to_index(sha) + offset = self._index.offset(index) + pack_data = self._pack.data() + next_index = min(self._index.size()-1, index+1) + next_offset = 0 + if next_index == index: + next_offset = len(pack_data) - self._pack.footer_size + else: + next_offset = self._index.offset(next_index) + # END get next offset + crc_value = self._index.crc(index) + + this_crc_value = 0 + crc_update = zlib.crc32 + + # create the current crc value, on the compressed object data + # Read it in chunks, without copying the data + cur_pos = offset + while cur_pos < next_offset: + rbound = min(cur_pos + chunk_size, next_offset) + size = rbound - cur_pos + crc_update(buffer(pack_data, cur_pos, size), this_crc_value) + cur_pos += size + # END window size loop + + assert this_crc_value == crc_value + return this_crc_value == crc_value + else: + shawriter = Sha1Writer() + stream = self._object(sha, as_stream=True) + # write a loose object, which is the basis for the sha + write_object(stream.type, stream.size, stream.read, shawriter.write) + + return shawriter.sha(as_hex=False) == sha + # END handle crc/sha verification + return True + def info_iter(self): """:return: Iterator over all objects in this pack. The iterator yields OInfo instances""" diff --git a/stream.py b/stream.py index 10a8e057c..903aeb109 100644 --- a/stream.py +++ b/stream.py @@ -474,7 +474,7 @@ class Sha1Writer(object): __slots__ = "sha1" def __init__(self): - self.sha1 = make_sha("") + self.sha1 = make_sha() #{ Stream Interface diff --git a/test/test_pack.py b/test/test_pack.py index 1f860961e..03b162178 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -10,10 +10,16 @@ ) from gitdb.pack import ( + PackEntity, PackIndexFile, PackFile ) + +from gitdb.fun import ( + delta_types, + ) from gitdb.util import to_bin_sha +from itertools import izip import os @@ -84,7 +90,7 @@ def _assert_pack_file(self, pack, version, size): # END get deltastream # read all - assert len(dstream.read()) + assert len(dstream.read()) == dstream.size # read chunks # NOTE: the current implementation is safe, it basically transfers @@ -109,8 +115,34 @@ def test_pack(self): # END for each pack to test def test_pack_entity(self): - # TODO: - pass + for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), + (self.packfile_v2_2, self.packindexfile_v2)): + packfile, version, size = packinfo + indexfile, version, size = indexinfo + print packfile + entity = PackEntity(packfile) + assert entity.pack().path() == packfile + assert entity.index().path() == indexfile + + count = 0 + for info, stream in izip(entity.info_iter(), entity.stream_iter()): + count += 1 + assert info.sha == stream.sha + assert len(info.sha) == 20 + assert info.type_id == stream.type_id + assert info.size == stream.size + + # we return fully resolved items, which is implied by the sha centric access + assert not info.type_id in delta_types + + # verify the stream + print info + assert entity.is_valid_stream(info.sha, use_crc=True) + #assert entity.is_valid_stream(info.sha, use_crc=False) + # END for each info, stream tuple + assert count == size + + # END for each entity def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets diff --git a/util.py b/util.py index f10b71b12..c460969ca 100644 --- a/util.py +++ b/util.py @@ -79,21 +79,6 @@ def make_sha(source=''): sha1 = sha.sha(source) return sha1 -def stream_copy(source, destination, chunk_size=512*1024): - """Copy all data from the source stream into the destination stream in chunks - of size chunk_size - - :return: amount of bytes written""" - br = 0 - while True: - chunk = source.read(chunk_size) - destination.write(chunk) - br += len(chunk) - if len(chunk) < chunk_size: - break - # END reading output stream - return br - def allocate_memory(size): """:return: a file-protocol accessible memory block of the given size""" try: From ecb18782c423bfdc45a474af2b7fbb61f62fa750 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 17:56:20 +0200 Subject: [PATCH 0024/1392] CRC verification already works for all packs, sha1 still needs some work, probably with deltified objects, there it shows whether we did it aaaaaall correctly ;) --- pack.py | 75 ++++++++++++++++++++++++++++++++++++----------- test/test_pack.py | 14 +++++---- 2 files changed, 67 insertions(+), 22 deletions(-) diff --git a/pack.py b/pack.py index b1f2f1e05..4003e2742 100644 --- a/pack.py +++ b/pack.py @@ -1,6 +1,7 @@ """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( - BadObject, + BadObject, + UnsupportedOperation ) from util import ( zlib, @@ -41,6 +42,8 @@ pack, ) +from itertools import izip +import array import os __all__ = ('PackIndexFile', 'PackFile', 'PackEntity') @@ -253,6 +256,21 @@ def indexfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of this index file""" return self._data[-20:] + def offsets(self): + """:return: sequence of all offsets in the order in which they were written + :note: return value can be random accessed, but may be immmutable""" + if self._version == 2: + # read stream to array, convert to tuple + a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears + a.fromstring(buffer(self._data, self._pack_offset, self._pack_64_offset - self._pack_offset)) + + # networkbyteorder to something array likes more + a.byteswap() + return a + else: + return tuple(self.offset(index) for index in xrange(self.size())) + # END handle version + def sha_to_index(self, sha): """ :return: index usable with the ``offset`` or ``entry`` method, or None @@ -419,11 +437,14 @@ def stream_iter(self, start_offset=0): #} END Read-Database like Interface -class PackEntity(object): +class PackEntity(LazyMixin): """Combines the PackIndexFile and the PackFile into one, allowing the actual objects to be resolved and iterated""" - __slots__ = ('_index', '_pack') + __slots__ = ( '_index', # our index file + '_pack', # our pack file + '_offset_map' # on demand dict mapping one offset to the next consecutive one + ) IndexFileCls = PackIndexFile PackFileCls = PackFile @@ -433,6 +454,28 @@ def __init__(self, pack_or_index_path): basename, ext = os.path.splitext(pack_or_index_path) self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + + def _set_cache_(self, attr): + # currently this can only be _offset_map + offsets_sorted = sorted(self._index.offsets()) + last_offset = len(self._pack.data()) - self._pack.footer_size + assert offsets_sorted, "Cannot handle empty indices" + + offset_map = None + if len(offsets_sorted) == 1: + offset_map = { offsets_sorted[0] : last_offset } + else: + iter_offsets = iter(offsets_sorted) + iter_offsets_plus_one = iter(offsets_sorted) + iter_offsets_plus_one.next() + consecutive = izip(iter_offsets, iter_offsets_plus_one) + + offset_map = dict(consecutive) + + # the last offset is not yet set + offset_map[offsets_sorted[-1]] = last_offset + # END handle offset amount + self._offset_map = offset_map def _sha_to_index(self, sha): """:return: index for the given sha, or raise""" @@ -537,33 +580,31 @@ def is_valid_stream(self, sha, use_crc=False): :raise UnsupportedOperation: If the index is version 1 only :raise BadObject: sha was not found""" if use_crc: + if self._index.version() < 2: + raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") + # END handle index version + index = self._sha_to_index(sha) offset = self._index.offset(index) - pack_data = self._pack.data() - next_index = min(self._index.size()-1, index+1) - next_offset = 0 - if next_index == index: - next_offset = len(pack_data) - self._pack.footer_size - else: - next_offset = self._index.offset(next_index) - # END get next offset + next_offset = self._offset_map[offset] crc_value = self._index.crc(index) - this_crc_value = 0 - crc_update = zlib.crc32 - # create the current crc value, on the compressed object data # Read it in chunks, without copying the data + crc_update = zlib.crc32 + pack_data = self._pack.data() cur_pos = offset + this_crc_value = 0 while cur_pos < next_offset: rbound = min(cur_pos + chunk_size, next_offset) size = rbound - cur_pos - crc_update(buffer(pack_data, cur_pos, size), this_crc_value) + this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) cur_pos += size # END window size loop - assert this_crc_value == crc_value - return this_crc_value == crc_value + # crc returns signed 32 bit numbers, the AND op forces it into unsigned + # mode ... wow, sneaky, from dulwich. + return (this_crc_value & 0xffffffff) == crc_value else: shawriter = Sha1Writer() stream = self._object(sha, as_stream=True) diff --git a/test/test_pack.py b/test/test_pack.py index 03b162178..d9823efa0 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -15,9 +15,8 @@ PackFile ) -from gitdb.fun import ( - delta_types, - ) +from gitdb.fun import delta_types +from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha from itertools import izip import os @@ -42,6 +41,7 @@ def _assert_index_file(self, index, version, size): assert len(index.indexfile_checksum()) == 20 assert index.version() == version assert index.size() == size + assert len(index.offsets()) == size # get all data of all objects for oidx in xrange(index.size()): @@ -137,8 +137,12 @@ def test_pack_entity(self): # verify the stream print info - assert entity.is_valid_stream(info.sha, use_crc=True) - #assert entity.is_valid_stream(info.sha, use_crc=False) + try: + assert entity.is_valid_stream(info.sha, use_crc=True) + except UnsupportedOperation: + pass + # END ignore version issues + assert entity.is_valid_stream(info.sha, use_crc=False) # END for each info, stream tuple assert count == size From 5af5cd919aea64aeb8be8a5dc38cc6a169878399 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 18:36:33 +0200 Subject: [PATCH 0025/1392] Sha1 verification works as well, forgot to fill in the base buffer for delta-application, and fixed the broken DeltaApplyReader's seek method --- fun.py | 2 +- pack.py | 1 + stream.py | 6 +++++- test/test_pack.py | 10 +++++++--- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/fun.py b/fun.py index 1466a78bb..dd9a53b5d 100644 --- a/fun.py +++ b/fun.py @@ -200,7 +200,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi if (cp_off + cp_size < cp_size or cp_off + cp_size > src_buf_size): break - twrite(src_buf[cp_off:cp_off+cp_size]) + twrite(buffer(src_buf, cp_off, cp_size)) elif c: twrite(db[i:i+c]) i += c diff --git a/pack.py b/pack.py index 4003e2742..ca9dac959 100644 --- a/pack.py +++ b/pack.py @@ -611,6 +611,7 @@ def is_valid_stream(self, sha, use_crc=False): # write a loose object, which is the basis for the sha write_object(stream.type, stream.size, stream.read, shawriter.write) + assert shawriter.sha(as_hex=False) == sha return shawriter.sha(as_hex=False) == sha # END handle crc/sha verification return True diff --git a/stream.py b/stream.py index 903aeb109..b0fa4acb5 100644 --- a/stream.py +++ b/stream.py @@ -358,6 +358,7 @@ def _set_cache_(self, attr): # Allocate private memory map big enough to hold the first base buffer # We need random access to it bbuf = allocate_memory(base_size) + stream_copy(self._bstream.read, bbuf.write, base_size, 256*mmap.PAGESIZE) # allocate memory map large enough for the largest (intermediate) target # We will use it as scratch space for all delta ops. If the final @@ -408,6 +409,8 @@ def read(self, count=0): bl = self._size - self._br # bytes left if count < 1 or count > bl: count = bl + # NOTE: we could check for certain size limits, and possibly + # return buffers instead of strings to prevent byte copying data = self._mm_target.read(count) self._br += len(data) return data @@ -418,7 +421,8 @@ def seek(self, offset, whence=os.SEEK_SET): if offset != 0 or whence != os.SEEK_SET: raise ValueError("Can only seek to position 0") # END handle offset - self._size + self._br = 0 + self._mm_target.seek(0) #{ Interface diff --git a/test/test_pack.py b/test/test_pack.py index d9823efa0..fbcf84e39 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -90,7 +90,13 @@ def _assert_pack_file(self, pack, version, size): # END get deltastream # read all - assert len(dstream.read()) == dstream.size + data = dstream.read() + assert len(data) == dstream.size + + # test seek + dstream.seek(0) + assert dstream.read() == data + # read chunks # NOTE: the current implementation is safe, it basically transfers @@ -119,7 +125,6 @@ def test_pack_entity(self): (self.packfile_v2_2, self.packindexfile_v2)): packfile, version, size = packinfo indexfile, version, size = indexinfo - print packfile entity = PackEntity(packfile) assert entity.pack().path() == packfile assert entity.index().path() == indexfile @@ -136,7 +141,6 @@ def test_pack_entity(self): assert not info.type_id in delta_types # verify the stream - print info try: assert entity.is_valid_stream(info.sha, use_crc=True) except UnsupportedOperation: From 325742cfe258436302fe0bb92462dc18a22261c7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 22:36:44 +0200 Subject: [PATCH 0026/1392] Implemented basic info and stream retrieval as well as pack file handling of PackedDB - its now operational. Next up is a performance test --- db/pack.py | 123 +++++++++++++++++++++++++++++++++++++++++-- pack.py | 30 +++++++---- test/db/test_pack.py | 45 ++++++++++++++-- test/test_pack.py | 14 +++-- 4 files changed, 193 insertions(+), 19 deletions(-) diff --git a/db/pack.py b/db/pack.py index a850e0fb2..f5ee04d6e 100644 --- a/db/pack.py +++ b/db/pack.py @@ -4,29 +4,95 @@ ObjectDBR ) +from gitdb.util import ( + to_bin_sha, + LazyMixin + ) + from gitdb.exc import ( + BadObject, UnsupportedOperation, ) +from gitdb.pack import PackEntity + +import os +import glob __all__ = ('PackedDB', ) -class PackedDB(FileDBBase, ObjectDBR): + +#{ Utilities + + +class PackedDB(FileDBBase, ObjectDBR, LazyMixin): """A database operating on a set of object packs""" + # sort the priority list every N queries + _sort_interval = 15 + def __init__(self, root_path): super(PackedDB, self).__init__(root_path) + # list of lists with three items: + # * hits - number of times the pack was hit with a request + # * entity - Pack entity instance + # * sha_to_index - PackIndexFile.sha_to_index method for direct cache query + # self._entities = list() # lazy loaded list + self._hit_count = 0 # amount of hits + self._st_mtime = 0 # last modification data of our root path + + def _set_cache_(self, attr): + # currently it can only be our _entities attribute + self._entities = list() + self.update_pack_entity_cache() + + def _sort_entities(self): + self._entities.sort(key=lambda l: l[0], reverse=True) + + def _pack_info(self, sha): + """:return: tuple(entity, index) for an item at the given sha + :param sha: 20 or 40 byte sha + :raise BadObject: + :note: This method is not thread-safe, but may be hit in multi-threaded + operation. The worst thing that can happen though is a counter that + was not incremented, or the list being in wrong order. So we safe + the time for locking here, lets see how that goes""" + # presort ? + if self._hit_count % self._sort_interval == 0: + self._sort_entities() + # END update sorting + + sha = to_bin_sha(sha) + for item in self._entities: + index = item[2](sha) + if index is not None: + item[0] += 1 # one hit for you + self._hit_count += 1 # general hit count + return (item[1], index) + # END index found in pack + # END for each item + # no hit, see whether we have to update packs + # NOTE: considering packs don't change very often, we safe this call + # and leave it to the super-caller to trigger that + raise BadObject(sha) #{ Object DB Read def has_object(self, sha): - raise NotImplementedError() + try: + self._pack_info(sha) + return True + except BadObject: + return False + # END exception handling def info(self, sha): - raise NotImplementedError() + entity, index = self._pack_info(sha) + return entity.info_at_index(index) def stream(self, sha): - raise NotImplementedError() + entity, index = self._pack_info(sha) + return entity.stream_at_index(index) #} END object db read @@ -39,6 +105,55 @@ def store(self, istream): raise UnsupportedOperation() def store_async(self, reader): + # TODO: add ObjectDBRW before implementing this raise NotImplementedError() #} END object db write + + + #{ Interface + + def update_pack_entity_cache(self, force=False): + """Update our cache with the acutally existing packs on disk. Add new ones, + and remove deleted ones. We keep the unchanged ones + :param force: If True, the cache will be updated even though the directory + does not appear to have changed according to its modification timestamp. + :return: True if the packs have been updated so there is new information, + False if there was no change to the pack database""" + stat = os.stat(self.root_path()) + if not force and stat.st_mtime <= self._st_mtime: + return False + # END abort early on no change + self._st_mtime = stat.st_mtime + + # packs are supposed to be prefixed with pack- by git-convention + # get all pack files, figure out what changed + pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) + our_pack_files = set(item[1].pack().path() for item in self._entities) + + # new packs + for pack_file in (pack_files - our_pack_files): + # init the hit-counter/priority with the size, a good measure for hit- + # probability. Its implemented so that only 12 bytes will be read + entity = PackEntity(pack_file) + self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) + # END for each new packfile + + # removed packs + for pack_file in (our_pack_files - pack_files): + del_index = -1 + for i, item in enumerate(self._entities): + if item[1].pack().path() == pack_file: + del_index = i + break + # END found index + # END for each entity + assert del_index != -1 + del(self._entities[del_index]) + # END for each removed pack + + # reinitialize prioritiess + self._sort_entities() + return True + + #} END interface diff --git a/pack.py b/pack.py index ca9dac959..4e70c7ce9 100644 --- a/pack.py +++ b/pack.py @@ -40,6 +40,7 @@ from struct import ( pack, + unpack, ) from itertools import izip @@ -196,7 +197,7 @@ def _offset_v2(self, i): # in the 64 bit region of the file. The current offset ( lower 31 bits ) # are the index into it if offset & 0x80000000: - offset = unpack_from(">Q", self._data, self._pack_64_offset + (self.offset & ~0x80000000) * 8)[0] + offset = unpack_from(">Q", self._data, self._pack_64_offset + (offset & ~0x80000000) * 8)[0] # END handle 64 bit offset return offset @@ -291,7 +292,7 @@ def sha_to_index(self, sha): elif not c: return mid else: - lo = mid + lo = mid + 1 # END handle midpoint # END bisect return None @@ -326,13 +327,15 @@ def _set_cache_(self, attr): fd = ldb.open() self._data = file_contents_ro(fd) ldb.rollback() - # TODO: figure out whether we should better keep the lock, or maybe - # add a .keep file instead ? - else: + # read the header information type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0) - assert type_id == "PACK", "Pack file format is invalid: %r" % type_id - assert self._version in (2, 3), "Cannot handle pack format version %i" % self._version + + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? + else: # must be '_size' or '_version' + # read header info - we do that just with a file stream + type_id, self._version, self._size = unpack(">4sLL", open(self._packpath).read(12)) # END handle header def _iter_objects(self, start_offset, as_stream=True): @@ -545,14 +548,23 @@ def info(self, sha): :param sha: 20 byte sha1 :raise BadObject: :return: OInfo instance, with 20 byte sha""" - return self._object(sha, as_stream=False) + return self._object(sha, False) def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha :param sha: 20 byte sha1 :raise BadObject: :return: OStream instance, with 20 byte sha""" - return self._object(sha, as_stream=True) + return self._object(sha, True) + + def info_at_index(self, index): + """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" + return self._object(None, False, index) + + def stream_at_index(self, index): + """As ``stream``, but uses a PackIndexFile compatible index to refer to the + object""" + return self._object(None, True, index) #} END Read-Database like Interface diff --git a/test/db/test_pack.py b/test/db/test_pack.py index 6faff4695..d37c886eb 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -1,12 +1,51 @@ from lib import * from gitdb.db import PackedDB - +from gitdb.test.lib import fixture_path + +import os +import random + class TestPackDB(TestDBBase): @with_rw_directory @with_packs_rw def test_writing(self, path): - ldb = PackedDB(path) - # TODO + pdb = PackedDB(path) + + # on demand, we init our pack cache + num_packs = 2 + assert len(pdb._entities) == num_packs + assert pdb._st_mtime != 0 + + # test pack directory changed: + # packs removed - rename a file, should affect the glob + pack_path = pdb._entities[0][1].pack().path() + new_pack_path = pack_path + "renamed" + os.rename(pack_path, new_pack_path) + pdb.update_pack_entity_cache(force=True) + assert len(pdb._entities) == num_packs - 1 + + # packs added + os.rename(new_pack_path, pack_path) + pdb.update_pack_entity_cache(force=True) + assert len(pdb._entities) == num_packs + # bang on the cache + # access the Entities directly, as there is no iteration interface + # yet ( or required for now ) + sha_list = list() + for entity in (item[1] for item in pdb._entities): + for index in xrange(entity.index().size()): + + sha_list.append(entity.index().sha(index)) + # END for each index + # END for each entity + + # hit all packs in random order + random.shuffle(sha_list) + + for sha in sha_list: + info = pdb.info(sha) + stream = pdb.stream(sha) + # END for each sha to query diff --git a/test/test_pack.py b/test/test_pack.py index fbcf84e39..6cfd784d4 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -5,9 +5,7 @@ with_packs_rw, fixture_path ) -from gitdb.stream import ( - DeltaApplyReader - ) +from gitdb.stream import DeltaApplyReader from gitdb.pack import ( PackEntity, @@ -15,6 +13,11 @@ PackFile ) +from gitdb.base import ( + OInfo, + OStream, + ) + from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha @@ -140,6 +143,11 @@ def test_pack_entity(self): # we return fully resolved items, which is implied by the sha centric access assert not info.type_id in delta_types + # try all calls + assert len(entity.collect_streams(info.sha)) + assert isinstance(entity.info(info.sha), OInfo) + assert isinstance(entity.stream(info.sha), OStream) + # verify the stream try: assert entity.is_valid_stream(info.sha, use_crc=True) From 8ab9b4f307a0672fb7b7ae29812b4ffd3d9dc8bc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 17 Jun 2010 23:36:53 +0200 Subject: [PATCH 0027/1392] PackedDB: added sha_iter and size methods, these should move to the ObjectDBR actually Added performance test, packed stream reading still runs into errors, which is interesting as it dealt with the sample packs very well before --- db/pack.py | 19 +++++++++- test/db/test_pack.py | 9 ++--- test/performance/lib.py | 1 + test/performance/test_db.py | 70 +++++++++++++++++++++++++++++++++---- 4 files changed, 84 insertions(+), 15 deletions(-) diff --git a/db/pack.py b/db/pack.py index f5ee04d6e..97890ee20 100644 --- a/db/pack.py +++ b/db/pack.py @@ -28,7 +28,9 @@ class PackedDB(FileDBBase, ObjectDBR, LazyMixin): """A database operating on a set of object packs""" # sort the priority list every N queries - _sort_interval = 15 + # Higher values are better, performance tests don't show this has + # any effect, but it should have one + _sort_interval = 500 def __init__(self, root_path): super(PackedDB, self).__init__(root_path) @@ -156,4 +158,19 @@ def update_pack_entity_cache(self, force=False): self._sort_entities() return True + def sha_iter(self): + """Return iterator yielding 20 byte shas for the packed objects in this data base""" + sha_list = list() + for entity in (item[1] for item in self._entities): + index = entity.index() + sha_by_index = index.sha + for index in xrange(index.size()): + yield sha_by_index(index) + # END for each index + # END for each entity + + def size(self): + """:return: amount of packed objects in this database""" + sizes = [item[1].index().size() for item in self._entities] + return reduce(lambda x,y: x+y, sizes) #} END interface diff --git a/test/db/test_pack.py b/test/db/test_pack.py index d37c886eb..89f73f036 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -34,13 +34,8 @@ def test_writing(self, path): # bang on the cache # access the Entities directly, as there is no iteration interface # yet ( or required for now ) - sha_list = list() - for entity in (item[1] for item in pdb._entities): - for index in xrange(entity.index().size()): - - sha_list.append(entity.index().sha(index)) - # END for each index - # END for each entity + sha_list = list(pdb.sha_iter()) + assert len(sha_list) == pdb.size() # hit all packs in random order random.shuffle(sha_list) diff --git a/test/performance/lib.py b/test/performance/lib.py index 03788c081..45e0ca53f 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -44,6 +44,7 @@ def setUpAll(cls): except AttributeError: pass cls.gitrepopath = resolve_or_fail(k_env_git_repo) + assert cls.gitrepopath.endswith('.git') #} END base classes diff --git a/test/performance/test_db.py b/test/performance/test_db.py index cd231b650..f6d855e20 100644 --- a/test/performance/test_db.py +++ b/test/performance/test_db.py @@ -1,15 +1,71 @@ """Performance tests for object store""" +from lib import ( + TestBigRepoR + ) + +from gitdb.db.pack import PackedDB import sys +import os from time import time - -from lib import ( - TestBigRepoR - ) +import random class TestGitDBPerformance(TestBigRepoR): - def test_random_access(self): - pass - # TODO: use the actual db for this + def test_pack_random_access(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + assert len(pdb._entities) > 1 + + # sha lookup + st = time() + sha_list = list(pdb.sha_iter()) + elapsed = time() - st + ns = len(sha_list) + print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) + + + # sha lookup: best-case and worst case access + pdb_pack_info = pdb._pack_info + access_times = list() + for rand in range(2): + if rand: + random.shuffle(sha_list) + # END shuffle shas + st = time() + for sha in sha_list: + pdb_pack_info(sha) + # END for each sha to look up + elapsed = time() - st + access_times.append(elapsed) + + # discard cache + del(pdb._entities) + pdb._entities + print >> sys.stderr, "PDB: looked up %i sha (random=%i) in %f s ( %f shas/s )" % (ns, rand, elapsed, ns / elapsed) + # END for each random mode + elapsed_order, elapsed_rand = access_times + + # well, its never really sequencial regarding the memory patterns, but it + # shows how well the prioriy cache performs + print >> sys.stderr, "PDB: sequential access is %f %% faster than random-access" % (100 - ((elapsed_order / elapsed_rand) * 100)) + + + # query info and streams only + max_items = 10000 # can wait longer when testing memory + for pdb_fun in (pdb.info, pdb.stream): + st = time() + for sha in sha_list[:max_items]: + pdb_fun(sha) + elapsed = time() - st + print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f info/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) + # END for each function + # retrieve stream and read all + max_items = 5000 + pdb_stream = pdb.stream + st = time() + for sha in sha_list[:max_items]: + stream = pdb_stream(sha) + stream.read() + elapsed = time() - st + print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes in %f s ( %f info/s )" % (max_items, elapsed, max_items / elapsed) From e9c5cf3df54d0879662194f22d43a984e89b2cdf Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 00:49:34 +0200 Subject: [PATCH 0028/1392] delta-apply now works after fixing a stupid type, instead of i + 1 I wrote i + i ... argh \! --- fun.py | 18 +++++++++++++----- stream.py | 24 +++++++++++------------- test/performance/test_db.py | 5 ++++- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/fun.py b/fun.py index dd9a53b5d..82b9373b2 100644 --- a/fun.py +++ b/fun.py @@ -155,7 +155,8 @@ def stream_copy(read, write, size, chunk_size): return dbw -def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file, + target_size): """Apply data from a delta buffer using a source buffer to the target file, which will be written to :param src_buf: random access data from which the delta was created @@ -163,6 +164,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data :param target_file: file like object to write the result to + :param target_size: size of the target buffer :note: transcribed to python from the similar routine in patch-delta.c""" i = 0 twrite = target_file.write @@ -180,7 +182,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi i += 1 if (c & 0x04): cp_off |= (ord(db[i]) << 16) - i += i + i += 1 if (c & 0x08): cp_off |= (ord(db[i]) << 24) i += 1 @@ -196,14 +198,20 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi if not cp_size: cp_size = 0x10000 - # maybe skip this check ? - if (cp_off + cp_size < cp_size or - cp_off + cp_size > src_buf_size): + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size or + cp_size > target_size): break twrite(buffer(src_buf, cp_off, cp_size)) + target_size -= cp_size elif c: + if c > target_size: + break twrite(db[i:i+c]) i += c + target_size -= c else: raise ValueError("unexpected delta opcode 0") # END handle command byte diff --git a/stream.py b/stream.py index b0fa4acb5..8c171b23a 100644 --- a/stream.py +++ b/stream.py @@ -327,19 +327,15 @@ def __init__(self, stream_list): def _set_cache_(self, attr): """If we are here, we apply the actual deltas""" - # fill in delta info structures, providing the source and target buffer - # sizes. - buffer_offset_list = list() - final_target_size = None + + # prefetch information + buffer_info_list = list() max_target_size = 0 for dstream in self._dstreams: buf = dstream.read(512) # read the header information + X offset, src_size = msb_size(buf) offset, target_size = msb_size(buf, offset) - if final_target_size is None: - final_target_size = target_size - # END set final target size - buffer_offset_list.append((buffer(buf, offset), offset)) + buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) max_target_size = max(max_target_size, target_size) # END for each delta stream @@ -358,7 +354,7 @@ def _set_cache_(self, attr): # Allocate private memory map big enough to hold the first base buffer # We need random access to it bbuf = allocate_memory(base_size) - stream_copy(self._bstream.read, bbuf.write, base_size, 256*mmap.PAGESIZE) + stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) # allocate memory map large enough for the largest (intermediate) target # We will use it as scratch space for all delta ops. If the final @@ -370,7 +366,8 @@ def _set_cache_(self, attr): # work on the op-codes to reconstruct everything. # For the actual copying, we use a seek and write pattern of buffer # slices. - for (dbuf, offset), dstream in reversed(zip(buffer_offset_list, self._dstreams)): + final_target_size = None + for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): # allocate a buffer to hold all delta data - fill in the data for # fast access. We do this as we know that reading individual bytes # from our stream would be slower than necessary ( although possible ) @@ -381,15 +378,16 @@ def _set_cache_(self, attr): # read the rest from the stream. The size we give is larger than necessary stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) - ################################################################ - apply_delta_data(bbuf, len(bbuf), ddata, len(ddata), tbuf) - ################################################################ + ####################################################################### + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf, target_size) + ####################################################################### # finally, swap out source and target buffers. The target is now the # base for the next delta to apply bbuf, tbuf = tbuf, bbuf bbuf.seek(0) tbuf.seek(0) + final_target_size = target_size # END for each delta to apply # its already seeked to 0, constrain it to the actual size diff --git a/test/performance/test_db.py b/test/performance/test_db.py index f6d855e20..3948003cc 100644 --- a/test/performance/test_db.py +++ b/test/performance/test_db.py @@ -63,9 +63,12 @@ def test_pack_random_access(self): # retrieve stream and read all max_items = 5000 pdb_stream = pdb.stream + total_size = 0 st = time() for sha in sha_list[:max_items]: stream = pdb_stream(sha) stream.read() + total_size += stream.size elapsed = time() - st - print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes in %f s ( %f info/s )" % (max_items, elapsed, max_items / elapsed) + total_kib = total_size / 1000 + print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) From fc6253d8428631029a9cb42830c9829692e92997 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 01:04:45 +0200 Subject: [PATCH 0029/1392] removed some extra checks in apply-delta which are indeed not required --- fun.py | 11 ++--------- stream.py | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/fun.py b/fun.py index 82b9373b2..d7e43717c 100644 --- a/fun.py +++ b/fun.py @@ -155,8 +155,7 @@ def stream_copy(read, write, size, chunk_size): return dbw -def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file, - target_size): +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): """Apply data from a delta buffer using a source buffer to the target file, which will be written to :param src_buf: random access data from which the delta was created @@ -164,7 +163,6 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data :param target_file: file like object to write the result to - :param target_size: size of the target buffer :note: transcribed to python from the similar routine in patch-delta.c""" i = 0 twrite = target_file.write @@ -201,17 +199,12 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi rbound = cp_off + cp_size if (rbound < cp_size or - rbound > src_buf_size or - cp_size > target_size): + rbound > src_buf_size): break twrite(buffer(src_buf, cp_off, cp_size)) - target_size -= cp_size elif c: - if c > target_size: - break twrite(db[i:i+c]) i += c - target_size -= c else: raise ValueError("unexpected delta opcode 0") # END handle command byte diff --git a/stream.py b/stream.py index 8c171b23a..6c388a96c 100644 --- a/stream.py +++ b/stream.py @@ -379,7 +379,7 @@ def _set_cache_(self, attr): stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) ####################################################################### - apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf, target_size) + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf) ####################################################################### # finally, swap out source and target buffers. The target is now the From ae6d08e1b63bf05ddcb3ee5fe027c5d829380afc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 10:10:28 +0200 Subject: [PATCH 0030/1392] Removed data-offset field from PackInfo as it is not needed in most cases. Instead, pack_at_offset returns the data-offset, slightly improving performance, and reducing memory demands --- base.py | 36 ++++++++++++++++-------------------- pack.py | 23 ++++++++++++----------- test/performance/test_db.py | 2 +- test/test_base.py | 2 -- test/test_pack.py | 1 - 5 files changed, 29 insertions(+), 35 deletions(-) diff --git a/base.py b/base.py index aa917858d..25968f371 100644 --- a/base.py +++ b/base.py @@ -64,8 +64,8 @@ class OPackInfo(tuple): location in the pack at which that actual data stream can be found.""" __slots__ = tuple() - def __new__(cls, packoffset, dataoffset, type, size): - return tuple.__new__(cls, (packoffset, dataoffset, type, size)) + def __new__(cls, packoffset, type, size): + return tuple.__new__(cls, (packoffset,type, size)) def __init__(self, *args): tuple.__init__(self) @@ -76,21 +76,17 @@ def __init__(self, *args): def pack_offset(self): return self[0] - @property - def data_offset(self): - return self[1] - @property def type(self): - return type_id_to_type_map[self[2]] + return type_id_to_type_map[self[1]] @property def type_id(self): - return self[2] + return self[1] @property def size(self): - return self[3] + return self[2] #} END interface @@ -102,13 +98,13 @@ class ODeltaPackInfo(OPackInfo): the pack offset of the base object""" __slots__ = tuple() - def __new__(cls, packoffset, dataoffset, type, size, delta_info): - return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info)) + def __new__(cls, packoffset, type, size, delta_info): + return tuple.__new__(cls, (packoffset, type, size, delta_info)) #{ Interface @property def delta_info(self): - return self[4] + return self[3] #} END interface @@ -142,17 +138,17 @@ class OPackStream(OPackInfo): is provided""" __slots__ = tuple() - def __new__(cls, packoffset, dataoffset, type, size, stream, *args): + def __new__(cls, packoffset, type, size, stream, *args): """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (packoffset, dataoffset, type, size, stream)) + return tuple.__new__(cls, (packoffset, type, size, stream)) #{ Stream Reader Interface def read(self, size=-1): - return self[4].read(size) + return self[3].read(size) @property def stream(self): - return self[4] + return self[3] #} END stream reader interface @@ -160,17 +156,17 @@ class ODeltaPackStream(ODeltaPackInfo): """Provides a stream outputting the uncompressed offset delta information""" __slots__ = tuple() - def __new__(cls, packoffset, dataoffset, type, size, delta_info, stream): - return tuple.__new__(cls, (packoffset, dataoffset, type, size, delta_info, stream)) + def __new__(cls, packoffset, type, size, delta_info, stream): + return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) #{ Stream Reader Interface def read(self, size=-1): - return self[5].read(size) + return self[4].read(size) @property def stream(self): - return self[5] + return self[4] #} END stream reader interface diff --git a/pack.py b/pack.py index 4e70c7ce9..1996d0389 100644 --- a/pack.py +++ b/pack.py @@ -55,7 +55,7 @@ def pack_object_at(data, offset, as_stream): """ - :return: PackInfo|PackStream + :return: Tuple(abs_data_offset, PackInfo|PackStream) an object of the correct type according to the type_id of the object. If as_stream is True, the object will contain a stream, allowing the data to be read decompressed. @@ -97,14 +97,14 @@ def pack_object_at(data, offset, as_stream): if as_stream: stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) if delta_info is None: - return OPackStream(offset, abs_data_offset, type_id, uncomp_size, stream) + return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: - return ODeltaPackStream(offset, abs_data_offset, type_id, uncomp_size, delta_info, stream) + return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) else: if delta_info is None: - return OPackInfo(offset, abs_data_offset, type_id, uncomp_size) + return abs_data_offset, OPackInfo(offset, type_id, uncomp_size) else: - return ODeltaPackInfo(offset, abs_data_offset, type_id, uncomp_size, delta_info) + return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) # END handle info # END handle stream @@ -278,6 +278,7 @@ def sha_to_index(self, sha): if the sha was not found in this pack index :param sha: 20 byte sha to lookup""" first_byte = ord(sha[0]) + get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: lo = self._fanout_table[first_byte-1] @@ -286,7 +287,7 @@ def sha_to_index(self, sha): # bisect until we have the sha while lo < hi: mid = (lo + hi) / 2 - c = cmp(sha, self.sha(mid)) + c = cmp(sha, get_sha(mid)) if c < 0: hi = mid elif not c: @@ -346,12 +347,12 @@ def _iter_objects(self, start_offset, as_stream=True): null = NullStream() while cur_offset < content_size: - ostream = pack_object_at(data, cur_offset, True) + data_offset, ostream = pack_object_at(data, cur_offset, True) # scrub the stream to the end - this decompresses the object, but yields # the amount of compressed bytes we need to get to the next offset stream_copy(ostream.read, null.write, ostream.size, chunk_size) - cur_offset += (ostream.data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() + cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() # if a stream is requested, reset it beforehand @@ -399,7 +400,7 @@ def collect_streams(self, offset): :param offset: specifies the first byte of the object within this pack""" out = list() while True: - ostream = pack_object_at(self._data, offset, True) + ostream = pack_object_at(self._data, offset, True)[1] out.append(ostream) if ostream.type_id == OFS_DELTA: offset = ostream.pack_offset - ostream.delta_info @@ -420,13 +421,13 @@ def info(self, offset): """Retrieve information about the object at the given file-absolute offset :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self.first_object_offset, False) + return pack_object_at(self._data, offset or self.first_object_offset, False)[1] def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self.first_object_offset, True) + return pack_object_at(self._data, offset or self.first_object_offset, True)[1] def stream_iter(self, start_offset=0): """:return: iterator yielding OPackStream compatible instances, allowing diff --git a/test/performance/test_db.py b/test/performance/test_db.py index 3948003cc..46db794fd 100644 --- a/test/performance/test_db.py +++ b/test/performance/test_db.py @@ -14,7 +14,6 @@ class TestGitDBPerformance(TestBigRepoR): def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - assert len(pdb._entities) > 1 # sha lookup st = time() @@ -72,3 +71,4 @@ def test_pack_random_access(self): elapsed = time() - st total_kib = total_size / 1000 print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) + diff --git a/test/test_base.py b/test/test_base.py index 524bf3054..c122ec4b1 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -35,14 +35,12 @@ def test_streams(self): assert pinfo.type == str_blob_type assert pinfo.type_id == blob_id assert pinfo.pack_offset == 0 - assert pinfo.data_offset == 1 dpinfo = ODeltaPackInfo(0, 1, blob_id, s, sha) assert dpinfo.type == str_blob_type assert dpinfo.type_id == blob_id assert dpinfo.delta_info == sha assert dpinfo.pack_offset == 0 - assert dpinfo.data_offset == 1 # test ostream diff --git a/test/test_pack.py b/test/test_pack.py index 6cfd784d4..6821097ad 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -72,7 +72,6 @@ def _assert_pack_file(self, pack, version, size): stream = pack.stream(obj.pack_offset) assert info.pack_offset == stream.pack_offset - assert info.data_offset == stream.data_offset assert info.type_id == stream.type_id assert hasattr(stream, 'read') From 4503a78e699cb42935f15baa46f5947a0020911f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 12:37:42 +0200 Subject: [PATCH 0031/1392] Added initial test for putting some often-called functions into an extension module, for now its only being built by a cheap hardcoded makefile. It shows that the performance gain is rather small, bottlenecks are attr accesses, so in fact the whole type wants to be put into C to get real performance. Its not really worth it for 25% I believe --- .gitignore | 2 ++ _fun.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ makefile | 12 +++++++ pack.py | 13 ++++++++ 4 files changed, 124 insertions(+) create mode 100644 _fun.c create mode 100644 makefile diff --git a/.gitignore b/.gitignore index 0d20b6487..1a9d961a7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ *.pyc +*.o +*.so diff --git a/_fun.c b/_fun.c new file mode 100644 index 000000000..ce9f25b16 --- /dev/null +++ b/_fun.c @@ -0,0 +1,97 @@ +#include +#include + +static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) +{ + const unsigned char *sha; + const unsigned int sha_len; + + // Note: self is only set if we are a c type. We emulate an instance method, + // hence we have to get the instance as 'first' argument + + // get instance and sha + PyObject* inst = 0; + if (!PyArg_ParseTuple(args, "Os#", &inst, &sha, &sha_len)) + return NULL; + + if (sha_len != 20) { + PyErr_SetString(PyExc_ValueError, "Sha is not 20 bytes long"); + return NULL; + } + + if( !inst){ + PyErr_SetString(PyExc_ValueError, "Cannot be called without self"); + return NULL; + } + + // read lo and hi bounds + PyObject* fanout_table = PyObject_GetAttrString(inst, "_fanout_table"); + if (!fanout_table){ + PyErr_SetString(PyExc_ValueError, "Couldn't obtain fanout table"); + return NULL; + } + + unsigned int lo = 0, hi = 0; + if (sha[0]){ + PyObject* item = PySequence_GetItem(fanout_table, (const Py_ssize_t)(sha[0]-1)); + lo = PyInt_AS_LONG(item); + Py_DECREF(item); + } + PyObject* item = PySequence_GetItem(fanout_table, (const Py_ssize_t)sha[0]); + hi = PyInt_AS_LONG(item); + Py_DECREF(item); + item = 0; + + Py_DECREF(fanout_table); + + // get sha query function + PyObject* get_sha = PyObject_GetAttrString(inst, "sha"); + if (!get_sha){ + PyErr_SetString(PyExc_ValueError, "Couldn't obtain sha method"); + return NULL; + } + + PyObject *sha_str = 0; + while (lo < hi) { + const int mid = (lo + hi)/2; + sha_str = PyObject_CallFunction(get_sha, "i", mid); + if (!sha_str) { + return NULL; + } + + // we really trust that string ... for speed + const int cmp = memcmp(PyString_AS_STRING(sha_str), sha, 20); + Py_DECREF(sha_str); + sha_str = 0; + + if (cmp < 0){ + lo = mid + 1; + } + else if (cmp > 0) { + hi = mid; + } + else { + Py_DECREF(get_sha); + return PyInt_FromLong(mid); + }// END handle comparison + }// END while lo < hi + + // nothing found, cleanup + Py_DECREF(get_sha); + Py_RETURN_NONE; +} + + +static PyMethodDef py_fun[] = { + { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, NULL }, + { NULL, NULL, 0, NULL } +}; + +void init_fun(void) +{ + PyObject *m; + + m = Py_InitModule3("_fun", py_fun, NULL); + if (m == NULL) + return; +} diff --git a/makefile b/makefile new file mode 100644 index 000000000..390289625 --- /dev/null +++ b/makefile @@ -0,0 +1,12 @@ + +_fun.o: _fun.c + gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -I/usr/include/python2.6 -c $< -o $@ + +_fun.so: _fun.o + gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions $^ -o $@ + +all: _fun.so + +clean: + -rm *.so + -rm *.o diff --git a/pack.py b/pack.py index 1996d0389..d6fe56823 100644 --- a/pack.py +++ b/pack.py @@ -23,6 +23,12 @@ msb_size ) +try: + from _fun import PackIndexFile_sha_to_index +except ImportError: + pass +# END try c module + from base import ( # Amazing ! OInfo, OStream, @@ -298,6 +304,13 @@ def sha_to_index(self, sha): # END bisect return None + if 'PackIndexFile_sha_to_index' in globals(): + # NOTE: Its just about 25% faster, the major bottleneck might be the attr + # accesses + def sha_to_index(self, sha): + return PackIndexFile_sha_to_index(self, sha) + # END redefine heavy-hitter with c version + #} END properties From 3cee78ed377b0a73febdbd772ddba2999313023e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 15:52:16 +0200 Subject: [PATCH 0032/1392] Added endurance run to check all objects in the git source repository, something like a primtive git-fsck, which indeed takes a while to run. Its useful to see the memory consumption, which must stay static in the domain of physical memory --- db/pack.py | 6 ++++- test/performance/{test_db.py => test_pack.py} | 26 ++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) rename test/performance/{test_db.py => test_pack.py} (74%) diff --git a/db/pack.py b/db/pack.py index 97890ee20..2332992f2 100644 --- a/db/pack.py +++ b/db/pack.py @@ -157,11 +157,15 @@ def update_pack_entity_cache(self, force=False): # reinitialize prioritiess self._sort_entities() return True + + def entities(self): + """:return: list of pack entities operated upon by this database""" + return [ item[1] for item in self._entities ] def sha_iter(self): """Return iterator yielding 20 byte shas for the packed objects in this data base""" sha_list = list() - for entity in (item[1] for item in self._entities): + for entity in self.entities(): index = entity.index() sha_by_index = index.sha for index in xrange(index.size()): diff --git a/test/performance/test_db.py b/test/performance/test_pack.py similarity index 74% rename from test/performance/test_db.py rename to test/performance/test_pack.py index 46db794fd..37abc1725 100644 --- a/test/performance/test_db.py +++ b/test/performance/test_pack.py @@ -10,7 +10,7 @@ from time import time import random -class TestGitDBPerformance(TestBigRepoR): +class TestPackedDBPerformance(TestBigRepoR): def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) @@ -22,7 +22,6 @@ def test_pack_random_access(self): ns = len(sha_list) print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) - # sha lookup: best-case and worst case access pdb_pack_info = pdb._pack_info access_times = list() @@ -39,7 +38,7 @@ def test_pack_random_access(self): # discard cache del(pdb._entities) - pdb._entities + pdb.entities() print >> sys.stderr, "PDB: looked up %i sha (random=%i) in %f s ( %f shas/s )" % (ns, rand, elapsed, ns / elapsed) # END for each random mode elapsed_order, elapsed_rand = access_times @@ -72,3 +71,24 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) + + print >> sys.stderr, "Endurance run: verify streaming of %i objects (crc and sha)" % ns + for crc in range(2): + count = 0 + st = time() + for entity in pdb.entities(): + pack_verify = entity.is_valid_stream + sha_by_index = entity.index().sha + for index in xrange(entity.index().size()): + try: + assert pack_verify(sha_by_index(index), use_crc=crc) + except UnsupportedOperation: + pass + # END ignore old indices + count += 1 + # END for each index + # END for each entity + elapsed = time() - st + print >> sys.stderr, "PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed) + # END for each verify mode + From 6fd8d7406028d603379dc23be0ab1403785f2cd3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 18 Jun 2010 19:57:28 +0200 Subject: [PATCH 0033/1392] Base implementation and stubs added for git-like db, as well as the reference db ( for the alternates implementation ) --- db/base.py | 73 +++++++++++++++++++++++++++++++++-- db/git.py | 68 +++++++++++++++++++------------- db/loose.py | 19 +++++++++ db/pack.py | 45 ++++++++++----------- db/ref.py | 61 ++++++++++++++++++++++++++++- test/db/lib.py | 1 + test/db/test_git.py | 9 +++++ test/db/test_loose.py | 5 +++ test/db/test_pack.py | 12 +++--- test/db/test_ref.py | 21 ++++++++++ test/performance/test_pack.py | 5 ++- util.py | 1 + 12 files changed, 258 insertions(+), 62 deletions(-) create mode 100644 test/db/test_git.py create mode 100644 test/db/test_ref.py diff --git a/db/base.py b/db/base.py index 2cda0ea0a..91e82fc0b 100644 --- a/db/base.py +++ b/db/base.py @@ -9,7 +9,7 @@ ) -__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB') +__all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') class ObjectDBR(object): @@ -66,6 +66,14 @@ def stream_async(self, reader): # base implementation just uses the stream method repeatedly task = ChannelThreadTask(reader, str(self.stream_async), self.stream) return pool.add_task(task) + + def size(self): + """:return: amount of objects in this database""" + raise NotImplementedError() + + def sha_iter(self): + """Return iterator yielding 20 byte shas for all objects in this data base""" + raise NotImplementedError() #} END query interface @@ -150,6 +158,63 @@ def db_path(self, rela_path): #} END interface -class CompoundDB(ObjectDBR): - """A database which delegates calls to sub-databases""" - # TODO +class CachingDB(object): + """A database which uses caches to speed-up access""" + + #{ Interface + def update_cache(self, force=False): + """Call this method if the underlying data changed to trigger an update + of the internal caching structures. + :param force: if True, the update must be performed. Otherwise the implementation + may decide not to perform an update if it thinks nothing has changed. + :return: True if an update was performed as something change indeed""" + + # END interface + + +class CompoundDB(ObjectDBR, LazyMixin, CachingDB): + """A database which delegates calls to sub-databases. + + Databases are stored in the lazy-loaded _dbs attribute. + Define _set_cache_ to update it with your databases""" + + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + + #{ ObjectDBR interface + + def has_object(self, sha): + raise NotImplementedError("To be implemented in subclass") + + def info(self, sha): + raise NotImplementedError("To be implemented in subclass") + + def stream(self, sha): + raise NotImplementedError() + + def size(self): + raise NotImplementedError() + + def sha_iter(self): + raise NotImplementedError() + + #} END object DBR Interface + + #{ Interface + + def databases(self): + """:return: tuple of database instances we use for lookups""" + return tuple(self._dbs) + + def update_cache(self, force=False): + stat = False + for db in self._dbs: + if isinstance(db, CachingDB): + stat |= db.update_cache(force) + # END if is caching db + # END for each database to update + return stat + #} END interface + + diff --git a/db/git.py b/db/git.py index 0488bc601..1953def17 100644 --- a/db/git.py +++ b/db/git.py @@ -1,33 +1,49 @@ - -from gitdb.base import ( - OInfo, - OStream - ) +from base import ( + CompoundDB, + FileDBBase, + ) from loose import LooseObjectDB +from pack import PackedDB +from ref import ReferenceDB + +from gitdb.util import LazyMixin +from gitdb.exc import InvalidDBRoot +import os -__all__ = ('GitObjectDB', ) +__all__ = ('GitDB', ) -#class GitObjectDB(CompoundDB, ObjectDBW): -class GitObjectDB(LooseObjectDB): - """A database representing the default git object store, which includes loose - objects, pack files and an alternates file +class GitDB(FileDBBase, CompoundDB): + """A git-style object database, which contains all objects in the 'objects' + subdirectory""" + # Configuration + PackDBCls = PackedDB + LooseDBCls = LooseObjectDB + ReferenceDBCls = ReferenceDB - It will create objects only in the loose object database. - :note: for now, we use the git command to do all the lookup, just until he - have packs and the other implementations - """ - def __init__(self, root_path, git): - """Initialize this instance with the root and a git command""" - super(GitObjectDB, self).__init__(root_path) - self._git = git + # Directories + packs_dir = 'packs' + loose_dir = '' + alternates_dir = os.path.join('info', 'alternates') + + def __init__(self, root_path): + """Initialize ourselves on a git objects directory""" + super(GitDB, self).__init__(root_path) - def info(self, sha): - t = self._git.get_object_header(sha) - return OInfo(*t) + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + for subpath, dbcls in ((self.packs_dir, self.PackDBCls), + (self.loose_dir, self.LooseDBCls), + (self.alternates_dir, self.ReferenceDBCls)): + path = self.db_path(subpath) + if os.path.exists(path): + self._dbs.append(dbcls(path)) + # END check path exists + # END for each db type + + # should have at least one subdb + if not self._dbs: + raise InvalidDBRoot(self.root_path()) + # END handle dbs - def stream(self, sha): - """For now, all lookup is done by git itself""" - t = self._git.stream_object_data(sha) - return OStream(*t) - diff --git a/db/loose.py b/db/loose.py index 109782fdc..b95d6f1c9 100644 --- a/db/loose.py +++ b/db/loose.py @@ -24,11 +24,13 @@ from gitdb.util import ( ENOENT, to_hex_sha, + hex_to_bin, exists, isdir, mkdir, rename, dirname, + basename, join ) @@ -186,4 +188,21 @@ def store(self, istream): istream.sha = sha return istream + + def sha_iter(self): + # find all files which look like an object, extract sha from there + for root, dirs, files in os.walk(self.root_path()): + root_base = basename(root) + if len(root_base) != 2: + continue + + for f in files: + if len(f) != 38: + continue + yield hex_to_bin(root_base + f) + # END for each file + # END for each walk iteration + + def size(self): + return len(tuple(self.sha_iter())) diff --git a/db/pack.py b/db/pack.py index 2332992f2..92fbc616b 100644 --- a/db/pack.py +++ b/db/pack.py @@ -1,7 +1,8 @@ """Module containing a database to deal with packs""" from base import ( FileDBBase, - ObjectDBR + ObjectDBR, + CachingDB ) from gitdb.util import ( @@ -18,13 +19,13 @@ import os import glob -__all__ = ('PackedDB', ) +__all__ = ('PackedDB', ) #{ Utilities -class PackedDB(FileDBBase, ObjectDBR, LazyMixin): +class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): """A database operating on a set of object packs""" # sort the priority list every N queries @@ -43,9 +44,10 @@ def __init__(self, root_path): self._st_mtime = 0 # last modification data of our root path def _set_cache_(self, attr): - # currently it can only be our _entities attribute - self._entities = list() - self.update_pack_entity_cache() + if attr == '_entities': + self._entities = list() + self.update_cache() + # END handle entities initialization def _sort_entities(self): self._entities.sort(key=lambda l: l[0], reverse=True) @@ -95,6 +97,20 @@ def info(self, sha): def stream(self, sha): entity, index = self._pack_info(sha) return entity.stream_at_index(index) + + def sha_iter(self): + sha_list = list() + for entity in self.entities(): + index = entity.index() + sha_by_index = index.sha + for index in xrange(index.size()): + yield sha_by_index(index) + # END for each index + # END for each entity + + def size(self): + sizes = [item[1].index().size() for item in self._entities] + return reduce(lambda x,y: x+y, sizes) #} END object db read @@ -115,7 +131,7 @@ def store_async(self, reader): #{ Interface - def update_pack_entity_cache(self, force=False): + def update_cache(self, force=False): """Update our cache with the acutally existing packs on disk. Add new ones, and remove deleted ones. We keep the unchanged ones :param force: If True, the cache will be updated even though the directory @@ -162,19 +178,4 @@ def entities(self): """:return: list of pack entities operated upon by this database""" return [ item[1] for item in self._entities ] - def sha_iter(self): - """Return iterator yielding 20 byte shas for the packed objects in this data base""" - sha_list = list() - for entity in self.entities(): - index = entity.index() - sha_by_index = index.sha - for index in xrange(index.size()): - yield sha_by_index(index) - # END for each index - # END for each entity - - def size(self): - """:return: amount of packed objects in this database""" - sizes = [item[1].index().size() for item in self._entities] - return reduce(lambda x,y: x+y, sizes) #} END interface diff --git a/db/ref.py b/db/ref.py index 2c63884bc..5db8d7a23 100644 --- a/db/ref.py +++ b/db/ref.py @@ -1,7 +1,64 @@ -from base import CompoundDB +from base import ( + CompoundDB, + ) +import os __all__ = ('CompoundDB', ) class ReferenceDB(CompoundDB): """A database consisting of database referred to in a file""" - + + # Configuration + # Specifies the object database to use for the paths found in the alternates + # file. If None, it defaults to the GitDB + ObjectDBCls = None + + def __init__(self, ref_file): + super(ReferenceDB, self).__init__() + self._ref_file = ref_file + + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + self._update_dbs_from_ref_file() + # END handle dbs + + def _update_dbs_from_ref_file(self): + dbcls = self.ObjectDBCls + if dbcls is None: + # late import + from git import GitDB + dbcls = GitDB + # END get db type + + # try to get as many as possible, don't fail if some are unavailable + ref_paths = list() + try: + ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] + except OSError: + pass + # END handle alternates + + ref_paths_set = set(ref_paths) + cur_ref_paths_set = set(db.root_path() for db in self._dbs) + + # remove existing + for path in (cur_ref_paths_set - ref_paths_set): + for i, db in enumerate(self._dbs[:]): + if db.root_path() == path: + del(self._dbs[i]) + continue + # END del matching db + # END for each path to remove + + # add new + # sort them to maintain order + added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) + for path in added_paths: + self._dbs.append(dbcls(path)) + # END for each path to add + + def update_cache(self, force=False): + # re-read alternates and update databases + self._update_dbs_from_ref_file() + return super(ReferenceDB, self).update_cache(force) diff --git a/test/db/lib.py b/test/db/lib.py index cf752741b..57f5eefc1 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -3,6 +3,7 @@ with_rw_directory, with_packs_rw, ZippedStoreShaWriter, + fixture_path, TestBase ) diff --git a/test/db/test_git.py b/test/db/test_git.py new file mode 100644 index 000000000..35a0d5bad --- /dev/null +++ b/test/db/test_git.py @@ -0,0 +1,9 @@ +from lib import * +from gitdb.db import GitDB + +class TestGitDB(TestBase): + + def test_reading(self): + ldb = GitDB(fixture_path('../../.git/objects') + self.fail("todo") + diff --git a/test/db/test_loose.py b/test/db/test_loose.py index 70cd7742c..536b02048 100644 --- a/test/db/test_loose.py +++ b/test/db/test_loose.py @@ -11,3 +11,8 @@ def test_writing(self, path): self._assert_object_writing(ldb) self._assert_object_writing_async(ldb) + # verify sha iteration and size + shas = list(ldb.sha_iter()) + assert shas and len(shas[0]) == 20 + + assert len(shas) == ldb.size() diff --git a/test/db/test_pack.py b/test/db/test_pack.py index 89f73f036..f347f408b 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -14,22 +14,22 @@ def test_writing(self, path): # on demand, we init our pack cache num_packs = 2 - assert len(pdb._entities) == num_packs + assert len(pdb.entities()) == num_packs assert pdb._st_mtime != 0 # test pack directory changed: # packs removed - rename a file, should affect the glob - pack_path = pdb._entities[0][1].pack().path() + pack_path = pdb.entities()[0].pack().path() new_pack_path = pack_path + "renamed" os.rename(pack_path, new_pack_path) - pdb.update_pack_entity_cache(force=True) - assert len(pdb._entities) == num_packs - 1 + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs - 1 # packs added os.rename(new_pack_path, pack_path) - pdb.update_pack_entity_cache(force=True) - assert len(pdb._entities) == num_packs + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs # bang on the cache # access the Entities directly, as there is no iteration interface diff --git a/test/db/test_ref.py b/test/db/test_ref.py new file mode 100644 index 000000000..3b027d701 --- /dev/null +++ b/test/db/test_ref.py @@ -0,0 +1,21 @@ +from lib import * +from gitdb.db import ReferenceDB + +class TestReferenceDB(TestBase): + + @with_rw_directory + def test_writing(self, path): + # TODO: setup alternate file + alternates = + ldb = ReferenceDB(path) + + # try empty, non-existing + + # add two, one is invalid + + # remove valid + + # add valid + + self.fail("todo") + diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index 37abc1725..8046c1f83 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -3,6 +3,7 @@ TestBigRepoR ) +from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB import sys @@ -39,7 +40,7 @@ def test_pack_random_access(self): # discard cache del(pdb._entities) pdb.entities() - print >> sys.stderr, "PDB: looked up %i sha (random=%i) in %f s ( %f shas/s )" % (ns, rand, elapsed, ns / elapsed) + print >> sys.stderr, "PDB: looked up %i sha in %i packs (random=%i) in %f s ( %f shas/s )" % (ns, len(pdb.entities()), rand, elapsed, ns / elapsed) # END for each random mode elapsed_order, elapsed_rand = access_times @@ -82,10 +83,10 @@ def test_pack_random_access(self): for index in xrange(entity.index().size()): try: assert pack_verify(sha_by_index(index), use_crc=crc) + count += 1 except UnsupportedOperation: pass # END ignore old indices - count += 1 # END for each index # END for each entity elapsed = time() - st diff --git a/util.py b/util.py index c460969ca..aa6db4088 100644 --- a/util.py +++ b/util.py @@ -57,6 +57,7 @@ def unpack_from(fmt, data, offset=0): isdir = os.path.isdir rename = os.rename dirname = os.path.dirname +basename = os.path.basename join = os.path.join read = os.read write = os.write From 92ca2e4ad606fbec7c934ad9e467a1b51fddcc92 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 20 Jun 2010 19:47:38 +0200 Subject: [PATCH 0034/1392] Implemented gitdb, it should be a fully functional git database with full read support, and the ability to write loose objects --- db/base.py | 36 ++++++++++++++++++++++++++++++------ db/git.py | 36 ++++++++++++++++++++++++++++++------ db/pack.py | 2 +- db/ref.py | 15 ++++++++++++--- test/db/lib.py | 2 +- test/db/test_git.py | 23 ++++++++++++++++++++--- test/db/test_ref.py | 42 +++++++++++++++++++++++++++++++++++++----- 7 files changed, 131 insertions(+), 25 deletions(-) diff --git a/db/base.py b/db/base.py index 91e82fc0b..35c20b7e1 100644 --- a/db/base.py +++ b/db/base.py @@ -1,13 +1,19 @@ """Contains implementations of database retrieveing objects""" from gitdb.util import ( pool, - join + join, + LazyMixin, + to_bin_sha ) +from gitdb.exc import BadObject + from async import ( ChannelThreadTask ) +from itertools import chain + __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') @@ -182,22 +188,40 @@ def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() + def _db_query(self, sha): + """:return: database containing the given 20 or 40 byte sha + :raise BadObject:""" + # most databases use binary representations, prevent converting + # it everytime a database is being queried + sha = to_bin_sha(sha) + for db in self._dbs: + if db.has_object(sha): + return db + # END for each database + raise BadObject(sha) + #{ ObjectDBR interface def has_object(self, sha): - raise NotImplementedError("To be implemented in subclass") + try: + self._db_query(sha) + return True + except BadObject: + return False + # END handle exceptions def info(self, sha): - raise NotImplementedError("To be implemented in subclass") + return self._db_query(sha).info(sha) def stream(self, sha): - raise NotImplementedError() + return self._db_query(sha).stream(sha) def size(self): - raise NotImplementedError() + """:return: total size of all contained databases""" + return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) def sha_iter(self): - raise NotImplementedError() + return chain(*(db.sha_iter() for db in self._dbs)) #} END object DBR Interface diff --git a/db/git.py b/db/git.py index 1953def17..ad9a613b3 100644 --- a/db/git.py +++ b/db/git.py @@ -1,7 +1,8 @@ from base import ( - CompoundDB, - FileDBBase, - ) + CompoundDB, + ObjectDBW, + FileDBBase + ) from loose import LooseObjectDB from pack import PackedDB @@ -13,7 +14,7 @@ __all__ = ('GitDB', ) -class GitDB(FileDBBase, CompoundDB): +class GitDB(FileDBBase, ObjectDBW, CompoundDB): """A git-style object database, which contains all objects in the 'objects' subdirectory""" # Configuration @@ -22,7 +23,7 @@ class GitDB(FileDBBase, CompoundDB): ReferenceDBCls = ReferenceDB # Directories - packs_dir = 'packs' + packs_dir = 'pack' loose_dir = '' alternates_dir = os.path.join('info', 'alternates') @@ -31,19 +32,42 @@ def __init__(self, root_path): super(GitDB, self).__init__(root_path) def _set_cache_(self, attr): - if attr == '_dbs': + if attr == '_dbs' or attr == '_loose_db': self._dbs = list() + loose_db = None for subpath, dbcls in ((self.packs_dir, self.PackDBCls), (self.loose_dir, self.LooseDBCls), (self.alternates_dir, self.ReferenceDBCls)): path = self.db_path(subpath) if os.path.exists(path): self._dbs.append(dbcls(path)) + if dbcls is self.LooseDBCls: + loose_db = self._dbs[-1] + # END remember loose db # END check path exists # END for each db type # should have at least one subdb if not self._dbs: raise InvalidDBRoot(self.root_path()) + # END handle error + + # we the first one should have the store method + assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" + + # finally set the value + self._loose_db = loose_db + # END handle dbs + #{ ObjectDBW interface + + def store(self, istream): + return self._loose_db.store(istream) + + def ostream(self): + return self._loose_db.ostream() + + def set_ostream(self, ostream): + return self._loose_db.set_ostream(ostream) + #} END objectdbw interface diff --git a/db/pack.py b/db/pack.py index 92fbc616b..af6f7ffd6 100644 --- a/db/pack.py +++ b/db/pack.py @@ -110,7 +110,7 @@ def sha_iter(self): def size(self): sizes = [item[1].index().size() for item in self._entities] - return reduce(lambda x,y: x+y, sizes) + return reduce(lambda x,y: x+y, sizes, 0) #} END object db read diff --git a/db/ref.py b/db/ref.py index 5db8d7a23..3a4813979 100644 --- a/db/ref.py +++ b/db/ref.py @@ -3,7 +3,7 @@ ) import os -__all__ = ('CompoundDB', ) +__all__ = ('ReferenceDB', ) class ReferenceDB(CompoundDB): """A database consisting of database referred to in a file""" @@ -35,7 +35,7 @@ def _update_dbs_from_ref_file(self): ref_paths = list() try: ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] - except OSError: + except (OSError, IOError): pass # END handle alternates @@ -55,7 +55,16 @@ def _update_dbs_from_ref_file(self): # sort them to maintain order added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) for path in added_paths: - self._dbs.append(dbcls(path)) + try: + db = dbcls(path) + # force an update to verify path + if isinstance(db, CompoundDB): + db.databases() + # END verification + self._dbs.append(db) + except Exception, e: + # ignore invalid paths or issues + pass # END for each path to add def update_cache(self, force=False): diff --git a/test/db/lib.py b/test/db/lib.py index 57f5eefc1..2c597fbd3 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -22,7 +22,7 @@ from cStringIO import StringIO -__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw' ) +__all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') class TestDBBase(TestBase): """Base class providing testing routines on databases""" diff --git a/test/db/test_git.py b/test/db/test_git.py index 35a0d5bad..4d463f1f7 100644 --- a/test/db/test_git.py +++ b/test/db/test_git.py @@ -1,9 +1,26 @@ from lib import * from gitdb.db import GitDB +from gitdb.base import OStream, OInfo -class TestGitDB(TestBase): +class TestGitDB(TestDBBase): def test_reading(self): - ldb = GitDB(fixture_path('../../.git/objects') - self.fail("todo") + gdb = GitDB(fixture_path('../../.git/objects')) + # we have packs and loose objects, alternates doesn't necessarily exist + assert 1 < len(gdb.databases()) < 4 + + # access should be possible + gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976" + assert isinstance(gdb.info(gitdb_sha), OInfo) + assert isinstance(gdb.stream(gitdb_sha), OStream) + assert gdb.size() > 200 + assert len(list(gdb.sha_iter())) == gdb.size() + + @with_rw_directory + def test_writing(self, path): + gdb = GitDB(path) + + # its possible to write objects + self._assert_object_writing(gdb) + self._assert_object_writing_async(gdb) diff --git a/test/db/test_ref.py b/test/db/test_ref.py index 3b027d701..68d9b8116 100644 --- a/test/db/test_ref.py +++ b/test/db/test_ref.py @@ -1,21 +1,53 @@ from lib import * from gitdb.db import ReferenceDB -class TestReferenceDB(TestBase): +import os + +class TestReferenceDB(TestDBBase): + + def make_alt_file(self, alt_path, alt_list): + """Create an alternates file which contains the given alternates. + The list can be empty""" + alt_file = open(alt_path, "wb") + for alt in alt_list: + alt_file.write(alt + "\n") + alt_file.close() @with_rw_directory def test_writing(self, path): - # TODO: setup alternate file - alternates = - ldb = ReferenceDB(path) + null_sha_bin = '\0' * 20 + null_sha_hex = "0" * 40 + + alt_path = os.path.join(path, 'alternates') + rdb = ReferenceDB(alt_path) + assert len(rdb.databases()) == 0 + assert rdb.size() == 0 + assert len(list(rdb.sha_iter())) == 0 # try empty, non-existing + assert not rdb.has_object(null_sha_hex) + assert not rdb.has_object(null_sha_bin) + + # setup alternate file # add two, one is invalid + own_repo_path = fixture_path('../../.git/objects') # use own repo + self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 1 + + # we should now find a default revision of ours + gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976" + assert rdb.has_object(gitdb_sha) # remove valid + self.make_alt_file(alt_path, ["just/one/invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 0 # add valid + self.make_alt_file(alt_path, [own_repo_path]) + rdb.update_cache() + assert len(rdb.databases()) == 1 - self.fail("todo") From 92e6770be0f65393199432dcfd24f3f1b10d015e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 22 Jun 2010 10:58:45 +0200 Subject: [PATCH 0035/1392] Added MemoryDB including initial test, moved ZippedShaWriter into stream module, it was just a test helper previously --- base.py | 10 ++++++ db/__init__.py | 1 + db/mem.py | 80 +++++++++++++++++++++++++++++++++++++++++++++ stream.py | 33 +++++++++++++++++++ test/db/lib.py | 29 +++++++++++++++- test/db/test_mem.py | 10 ++++++ test/lib.py | 27 +++------------ test/test_base.py | 4 +-- 8 files changed, 169 insertions(+), 25 deletions(-) create mode 100644 db/mem.py create mode 100644 test/db/test_mem.py diff --git a/base.py b/base.py index 25968f371..0f4e63176 100644 --- a/base.py +++ b/base.py @@ -41,6 +41,16 @@ def __init__(self, *args): def sha(self): return self[0] + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return to_hex_sha(self[0]) + + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return to_bin_sha(self[0]) + @property def type(self): return self[1] diff --git a/db/__init__.py b/db/__init__.py index 05d9b21b3..85a0a6874 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -1,6 +1,7 @@ from base import * from loose import * +from mem import * from pack import * from git import * from ref import * diff --git a/db/mem.py b/db/mem.py new file mode 100644 index 000000000..3bb6a339d --- /dev/null +++ b/db/mem.py @@ -0,0 +1,80 @@ +"""Contains the MemoryDatabase implementation""" +from loose import LooseObjectDB +from base import ( + ObjectDBR, + ObjectDBW + ) + +from gitdb.base import OStream +from gitdb.util import to_bin_sha +from gitdb.exc import ( + BadObject, + UnsupportedOperation + ) +from gitdb.stream import ( + ZippedStoreShaWriter, + DecompressMemMapReader, + ) + +__all__ = ("MemoryDB", ) + +class MemoryDB(ObjectDBR, ObjectDBW): + """A memory database stores everything to memory, providing fast IO and object + retrieval. It should be used to buffer results and obtain SHAs before writing + it to the actual physical storage, as it allows to query whether object already + exists in the target storage before introducing actual IO + + :note: memory is currently not threadsafe, hence the async methods cannot be used + for storing""" + + def __init__(self): + super(MemoryDB, self).__init__() + self._db = LooseObjectDB("path/doesnt/matter") + + # maps 20 byte shas to their OStream objects + self._cache = dict() + + def set_ostream(self, stream): + raise UnsupportedOperation("MemoryDB's always stream into memory") + + def store(self, istream): + zstream = ZippedStoreShaWriter() + self._db.set_ostream(zstream) + + istream = self._db.store(istream) + zstream.close() # close to flush + zstream.seek(0) + + # don't provide a size, the stream is written in object format, hence the + # header needs decompression + decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) + self._cache[istream.binsha] = OStream(istream.sha, istream.type, istream.size, decomp_stream) + + return istream + + def store_async(self, reader): + raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") + + def has_object(self, sha): + return to_bin_sha(sha) in self._cache + + def info(self, sha): + # we always return streams, which are infos as well + return self.stream(sha) + + def stream(self, sha): + sha = to_bin_sha(sha) + try: + ostream = self._cache[sha] + # rewind stream for the next one to read + ostream.stream.seek(0) + return ostream + except KeyError: + raise BadObject(sha) + # END exception handling + + def size(self): + return len(self._cache) + + def sha_iter(self): + return self._cache.iterkeys() diff --git a/stream.py b/stream.py index 6c388a96c..8b7e981b0 100644 --- a/stream.py +++ b/stream.py @@ -499,6 +499,39 @@ def sha(self, as_hex = False): #} END interface + +class ZippedStoreShaWriter(Sha1Writer): + """Remembers everything someone writes to it and generates a sha""" + __slots__ = ('buf', 'zip') + def __init__(self): + Sha1Writer.__init__(self) + self.buf = StringIO() + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) + + def __getattr__(self, attr): + return getattr(self.buf, attr) + + def write(self, data): + alen = Sha1Writer.write(self, data) + self.buf.write(self.zip.compress(data)) + return alen + + def close(self): + self.buf.write(self.zip.flush()) + + def seek(self, offset, whence=os.SEEK_SET): + """Seeking currently only supports to rewind written data + Multiple writes are not supported""" + if offset != 0 or whence != os.SEEK_SET: + raise ValueError("Can only seek to position 0") + # END handle offset + self.buf.seek(0) + + def getvalue(self): + """:return: string value from the current stream position to the end""" + return self.buf.getvalue() + + class FDCompressedSha1Writer(Sha1Writer): """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor diff --git a/test/db/lib.py b/test/db/lib.py index 2c597fbd3..b22f9a2b8 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -20,6 +20,7 @@ from async import IteratorReader from cStringIO import StringIO +from struct import pack __all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') @@ -29,9 +30,35 @@ class TestDBBase(TestBase): # data two_lines = "1234\nhello world" - all_data = (two_lines, ) + def _assert_object_writing_simple(self, db): + # write a bunch of objects and query their streams and info + null_objs = db.size() + ni = 250 + for i in xrange(ni): + data = pack(">L", i) + istream = IStream(str_blob_type, len(data), StringIO(data)) + new_istream = db.store(istream) + assert new_istream is istream + assert db.has_object(istream.sha) + + info = db.info(istream.sha) + assert isinstance(info, OInfo) + assert info.type == istream.type and info.size == istream.size + + stream = db.stream(istream.sha) + assert isinstance(stream, OStream) + assert stream.sha == info.sha and stream.type == info.type + assert stream.read() == data + # END for each item + + assert db.size() == null_objs + ni + shas = list(db.sha_iter()) + assert len(shas) == db.size() + assert len(shas[0]) == 20 + + def _assert_object_writing(self, db): """General tests to verify object writing, compatible to ObjectDBW :note: requires write access to the database""" diff --git a/test/db/test_mem.py b/test/db/test_mem.py new file mode 100644 index 000000000..9e7c1190e --- /dev/null +++ b/test/db/test_mem.py @@ -0,0 +1,10 @@ +from lib import * +from gitdb.db import MemoryDB + +class TestMemoryDB(TestDBBase): + + def test_writing(self): + mdb = MemoryDB() + + # write data + self._assert_object_writing_simple(mdb) diff --git a/test/lib.py b/test/lib.py index 6b25876d6..78817fea9 100644 --- a/test/lib.py +++ b/test/lib.py @@ -2,7 +2,11 @@ from gitdb import ( OStream, ) -from gitdb.stream import Sha1Writer +from gitdb.stream import ( + Sha1Writer, + ZippedStoreShaWriter + ) + from gitdb.util import zlib import sys @@ -140,26 +144,5 @@ def _assert(self): assert self.args assert self.myarg - -class ZippedStoreShaWriter(Sha1Writer): - """Remembers everything someone writes to it""" - __slots__ = ('buf', 'zip') - def __init__(self): - Sha1Writer.__init__(self) - self.buf = StringIO() - self.zip = zlib.compressobj(1) # fastest - - def __getattr__(self, attr): - return getattr(self.buf, attr) - - def write(self, data): - alen = Sha1Writer.write(self, data) - self.buf.write(self.zip.compress(data)) - return alen - - def close(self): - self.buf.write(self.zip.flush()) - - #} END stream utilitiess diff --git a/test/test_base.py b/test/test_base.py index c122ec4b1..8d8bcc944 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -31,12 +31,12 @@ def test_streams(self): # test pack info # provides type_id - pinfo = OPackInfo(0, 1, blob_id, s) + pinfo = OPackInfo(0, blob_id, s) assert pinfo.type == str_blob_type assert pinfo.type_id == blob_id assert pinfo.pack_offset == 0 - dpinfo = ODeltaPackInfo(0, 1, blob_id, s, sha) + dpinfo = ODeltaPackInfo(0, blob_id, s, sha) assert dpinfo.type == str_blob_type assert dpinfo.type_id == blob_id assert dpinfo.delta_info == sha From 9b53ab02cb44571e6167a125a5296b7c3395563f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 22 Jun 2010 11:45:26 +0200 Subject: [PATCH 0036/1392] MemoryDB: Implemented direct stream copy, allowing to flush memory db content into any other object db for permanent storage --- db/loose.py | 24 ++++++++++++++++----- db/mem.py | 33 ++++++++++++++++++++++++++++- stream.py | 51 ++++++++++++++++++++++++++++++++------------- test/db/test_mem.py | 20 ++++++++++++++++-- test/test_stream.py | 4 ++-- 5 files changed, 107 insertions(+), 25 deletions(-) diff --git a/db/loose.py b/db/loose.py index b95d6f1c9..9bcbd6aac 100644 --- a/db/loose.py +++ b/db/loose.py @@ -13,6 +13,7 @@ from gitdb.stream import ( DecompressMemMapReader, FDCompressedSha1Writer, + FDStream, Sha1Writer ) @@ -43,6 +44,7 @@ import tempfile import mmap +import sys import os @@ -153,13 +155,20 @@ def store(self, istream): if writer is None: # open a tmp file to write the data to fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - writer = FDCompressedSha1Writer(fd) + + if istream.sha is None: + writer = FDCompressedSha1Writer(fd) + else: + writer = FDStream(fd) + # END handle direct stream copies # END handle custom writer try: try: if istream.sha is not None: - stream_copy(istream.read, writer.write, istream.size, self.stream_chunk_size) + # copy as much as possible, the actual uncompressed item size might + # be smaller than the compressed version + stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) else: # write object with header, we have to make a new one write_object(istream.type, istream.size, istream.read, writer.write, @@ -175,10 +184,15 @@ def store(self, istream): writer.close() # END assure target stream is closed - sha = istream.sha or writer.sha(as_hex=True) + hexsha = None + if istream.sha: + hexsha = istream.hexsha + else: + hexsha = writer.sha(as_hex=True) + # END handle sha if tmp_path: - obj_path = self.db_path(self.object_path(sha)) + obj_path = self.db_path(self.object_path(hexsha)) obj_dir = dirname(obj_path) if not isdir(obj_dir): mkdir(obj_dir) @@ -186,7 +200,7 @@ def store(self, istream): rename(tmp_path, obj_path) # END handle dry_run - istream.sha = sha + istream.sha = hexsha return istream def sha_iter(self): diff --git a/db/mem.py b/db/mem.py index 3bb6a339d..9e3d3972d 100644 --- a/db/mem.py +++ b/db/mem.py @@ -5,7 +5,11 @@ ObjectDBW ) -from gitdb.base import OStream +from gitdb.base import ( + OStream, + IStream, + ) + from gitdb.util import to_bin_sha from gitdb.exc import ( BadObject, @@ -16,6 +20,8 @@ DecompressMemMapReader, ) +from cStringIO import StringIO + __all__ = ("MemoryDB", ) class MemoryDB(ObjectDBR, ObjectDBW): @@ -78,3 +84,28 @@ def size(self): def sha_iter(self): return self._cache.iterkeys() + + + #{ Interface + def stream_copy(self, sha_iter, odb): + """Copy the streams as identified by sha's yielded by sha_iter into the given odb + The streams will be copied directly + :note: the object will only be written if it did not exist in the target db + :return: amount of streams actually copied into odb. If smaller than the amount + of input shas, one or more objects did already exist in odb""" + count = 0 + for sha in sha_iter: + if odb.has_object(sha): + continue + # END check object existance + + ostream = self.stream(sha) + # compressed data including header + sio = StringIO(ostream.stream.data()) + istream = IStream(ostream.type, ostream.size, sio, sha) + + odb.store(istream) + count += 1 + # END for each sha + return count + #} END interface diff --git a/stream.py b/stream.py index 8b7e981b0..57a5a194f 100644 --- a/stream.py +++ b/stream.py @@ -25,21 +25,6 @@ #{ RO Streams -class NullStream(object): - """A stream that does nothing but providing a stream interface. - Use it like /dev/null""" - __slots__ = tuple() - - def read(self, size=0): - return '' - - def close(self): - pass - - def write(self, data): - return len(data) - - class DecompressMemMapReader(LazyMixin): """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand @@ -113,6 +98,8 @@ def _parse_header_info(self): return type, size + #{ Interface + @classmethod def new(self, m, close_on_deletion=False): """Create a new DecompressMemMapReader instance for acting as a read-only stream @@ -125,6 +112,10 @@ def new(self, m, close_on_deletion=False): type, size = inst._parse_header_info() return type, size, inst + def data(self): + """:return: random access compatible data we are working on""" + return self._m + def compressed_bytes_read(self): """:return: number of compressed bytes read. This includes the bytes it took to decompress the header ( if there was one )""" @@ -171,6 +162,8 @@ def compressed_bytes_read(self): # from the count already return self._cbr + #} END interface + def seek(self, offset, whence=os.SEEK_SET): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" @@ -567,4 +560,32 @@ def close(self): #} END stream interface +class FDStream(object): + """Simple wrapper around a file descriptor""" + __slots__ = "_fd" + def __init__(self, fd): + self._fd = fd + + def write(self, data): + return write(self._fd, data) + + def close(self): + close(self._fd) + + + +class NullStream(object): + """A stream that does nothing but providing a stream interface. + Use it like /dev/null""" + __slots__ = tuple() + + def read(self, size=0): + return '' + + def close(self): + pass + + def write(self, data): + return len(data) + #} END W streams diff --git a/test/db/test_mem.py b/test/db/test_mem.py index 9e7c1190e..4a9b7ee12 100644 --- a/test/db/test_mem.py +++ b/test/db/test_mem.py @@ -1,10 +1,26 @@ from lib import * -from gitdb.db import MemoryDB +from gitdb.db import ( + MemoryDB, + LooseObjectDB + ) class TestMemoryDB(TestDBBase): - def test_writing(self): + @with_rw_directory + def test_writing(self, path): mdb = MemoryDB() # write data self._assert_object_writing_simple(mdb) + + # test stream copy + ldb = LooseObjectDB(path) + assert ldb.size() == 0 + num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) + assert num_streams_copied == mdb.size() + + assert ldb.size() == mdb.size() + for sha in mdb.sha_iter(): + assert ldb.has_object(sha) + assert ldb.stream(sha).read() == mdb.stream(sha).read() + # END verify objects where copied and are equal diff --git a/test/test_stream.py b/test/test_stream.py index 41f2b235a..859920719 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -50,7 +50,7 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): # END handle rest if isinstance(stream, DecompressMemMapReader): - assert len(stream._m) == stream.compressed_bytes_read() + assert len(stream.data()) == stream.compressed_bytes_read() # END handle special type rewind_stream(stream) @@ -60,7 +60,7 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): assert rdata == cdata if isinstance(stream, DecompressMemMapReader): - assert len(stream._m) == stream.compressed_bytes_read() + assert len(stream.data()) == stream.compressed_bytes_read() # END handle special type def test_decompress_reader(self): From 9e6e7d7f1f624143ef8e8fc04e55e5ca277f43ab Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Jun 2010 01:09:18 +0200 Subject: [PATCH 0037/1392] Removed redunant code due to parital reimplementation of a file descriptor stream wrapper --- db/loose.py | 14 +++++++------- stream.py | 29 ++++++++++++++++++++++++----- test/lib.py | 11 ++++++----- test/test_util.py | 10 ++++++++++ util.py | 43 +++++++++++-------------------------------- 5 files changed, 58 insertions(+), 49 deletions(-) diff --git a/db/loose.py b/db/loose.py index 9bcbd6aac..f97b98a2b 100644 --- a/db/loose.py +++ b/db/loose.py @@ -174,15 +174,15 @@ def store(self, istream): write_object(istream.type, istream.size, istream.read, writer.write, chunk_size=self.stream_chunk_size) # END handle direct stream copies - except: + finally: if tmp_path: - os.remove(tmp_path) - raise - # END assure tmpfile removal on error - finally: + writer.close() + # END assure target stream is closed + except: if tmp_path: - writer.close() - # END assure target stream is closed + os.remove(tmp_path) + raise + # END assure tmpfile removal on error hexsha = None if istream.sha: diff --git a/stream.py b/stream.py index 57a5a194f..3f5d0373a 100644 --- a/stream.py +++ b/stream.py @@ -560,19 +560,38 @@ def close(self): #} END stream interface + class FDStream(object): - """Simple wrapper around a file descriptor""" - __slots__ = "_fd" + """A simple wrapper providing the most basic functions on a file descriptor + with the fileobject interface. Cannot use os.fdopen as the resulting stream + takes ownership""" + __slots__ = ("_fd", '_pos') def __init__(self, fd): self._fd = fd + self._pos = 0 def write(self, data): - return write(self._fd, data) + self._pos += len(data) + os.write(self._fd, data) + + def read(self, count=0): + if count == 0: + count = os.path.getsize(self._filepath) + # END handle read everything + + bytes = os.read(self._fd, count) + self._pos += len(bytes) + return bytes + + def fileno(self): + return self._fd + + def tell(self): + return self._pos def close(self): close(self._fd) - - + class NullStream(object): """A stream that does nothing but providing a stream interface. diff --git a/test/lib.py b/test/lib.py index 78817fea9..742aa7f5c 100644 --- a/test/lib.py +++ b/test/lib.py @@ -38,11 +38,12 @@ def wrapper(self): path = tempfile.mktemp(prefix=func.__name__) os.mkdir(path) try: - return func(self, path) - except Exception: - print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) - raise - else: + try: + return func(self, path) + except Exception: + print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + raise + finally: shutil.rmtree(path) # END handle exception # END wrapper diff --git a/test/test_util.py b/test/test_util.py index 2272b53e5..6a389d27c 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -88,4 +88,14 @@ def test_lockedfd(self): finally: os.remove(my_file) # END final cleanup + + # try non-existing file for reading + lfd = LockedFD(tempfile.mktemp()) + try: + lfd.open(write=False) + except OSError: + assert not os.path.exists(lfd._lockfilepath()) + else: + self.fail("expected OSError") + # END handle exceptions diff --git a/util.py b/util.py index aa6db4088..9d1a96900 100644 --- a/util.py +++ b/util.py @@ -161,35 +161,6 @@ def _set_cache_(self, attr): in the single attribute.""" pass - -class FDStreamWrapper(object): - """A simple wrapper providing the most basic functions on a file descriptor - with the fileobject interface. Cannot use os.fdopen as the resulting stream - takes ownership""" - __slots__ = ("_fd", '_pos') - def __init__(self, fd): - self._fd = fd - self._pos = 0 - - def write(self, data): - self._pos += len(data) - os.write(self._fd, data) - - def read(self, count=0): - if count == 0: - count = os.path.getsize(self._filepath) - # END handle read everything - - bytes = os.read(self._fd, count) - self._pos += len(bytes) - return bytes - - def fileno(self): - return self._fd - - def tell(self): - return self._pos - class LockedFD(object): """This class facilitates a safe read and write operation to a file on disk. @@ -240,7 +211,7 @@ def open(self, write=False, stream=False): binary = getattr(os, 'O_BINARY', 0) lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary try: - fd = os.open(self._lockfilepath(), lockmode) + fd = os.open(self._lockfilepath(), lockmode, 0600) if not write: os.close(fd) else: @@ -253,11 +224,19 @@ def open(self, write=False, stream=False): # open actual file if required if self._fd is None: # we could specify exlusive here, as we obtained the lock anyway - self._fd = os.open(self._filepath, os.O_RDONLY | binary) + try: + self._fd = os.open(self._filepath, os.O_RDONLY | binary) + except: + # assure we release our lockfile + os.remove(self._lockfilepath()) + raise + # END handle lockfile # END open descriptor for reading if stream: - return FDStreamWrapper(self._fd) + # need delayed import + from stream import FDStream + return FDStream(self._fd) else: return self._fd # END handle stream From 09275ad268e5459c8edff2cccafd0a43947cabdb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Jun 2010 11:45:09 +0200 Subject: [PATCH 0038/1392] Index and PackFiles do not obtain a lock anymore before reading the files in question - this won't work on read-only alternate repositories anyway, and shouldn't be necessary considering a pack is immutable --- db/loose.py | 5 +++-- pack.py | 16 ++++++---------- util.py | 19 ++++++++++++++++++- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/db/loose.py b/db/loose.py index f97b98a2b..c9ea3a038 100644 --- a/db/loose.py +++ b/db/loose.py @@ -23,6 +23,7 @@ ) from gitdb.util import ( + file_contents_ro_filepath, ENOENT, to_hex_sha, hex_to_bin, @@ -100,12 +101,12 @@ def _map_loose_object(self, sha): :raise BadObject: if object could not be located""" db_path = self.db_path(self.object_path(to_hex_sha(sha))) try: - fd = os.open(db_path, os.O_RDONLY|self._fd_open_flags) + return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) except OSError,e: if e.errno != ENOENT: # try again without noatime try: - fd = os.open(db_path, os.O_RDONLY) + return file_contents_ro_filepath(db_path) except OSError: raise BadObject(to_hex_sha(sha)) # didn't work because of our flag, don't try it again diff --git a/pack.py b/pack.py index d6fe56823..c66d6717e 100644 --- a/pack.py +++ b/pack.py @@ -5,10 +5,9 @@ ) from util import ( zlib, - LockedFD, LazyMixin, unpack_from, - file_contents_ro, + file_contents_ro_filepath, ) from fun import ( @@ -140,10 +139,10 @@ def _set_cache_(self, attr): elif attr == "_packfile_checksum": self._packfile_checksum = self._data[-20:] elif attr == "_data": - lfd = LockedFD(self._indexpath) - fd = lfd.open() - self._data = file_contents_ro(fd) - lfd.rollback() + # Note: We don't lock the file when reading as we cannot be sure + # that we can actually write to the location - it could be a read-only + # alternate for instance + self._data = file_contents_ro_filepath(self._indexpath) else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties @@ -337,10 +336,7 @@ def __init__(self, packpath): def _set_cache_(self, attr): if attr == '_data': - ldb = LockedFD(self._packpath) - fd = ldb.open() - self._data = file_contents_ro(fd) - ldb.rollback() + self._data = file_contents_ro_filepath(self._packpath) # read the header information type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0) diff --git a/util.py b/util.py index 9d1a96900..f0bf3e60f 100644 --- a/util.py +++ b/util.py @@ -114,7 +114,24 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): if stream: return cStringIO.StringIO(contents) return contents - + +def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): + """Get the file contents at filepath as fast as possible + :return: random access compatible memory of the given filepath + :param stream: see ``file_contents_ro`` + :param allow_mmap: see ``file_contents_ro`` + :param flags: additional flags to pass to os.open + :raise OSError: If the file could not be opened + :note: for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object + databases anyway, and they use it with the help of the ``flags`` parameter""" + fd = os.open(filepath, os.O_RDONLY|flags) + try: + return file_contents_ro(fd, stream, allow_mmap) + finally: + close(fd) + # END assure file is closed + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: From d3a0037dd5a11459985e7dc4b6819f6292f20c13 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Jun 2010 15:28:22 +0200 Subject: [PATCH 0039/1392] Fixed critical issues with incorrect permissions set on files the db has written - it was only rw-- for the user that wrote them, but should be readable by everyone by default --- db/loose.py | 5 +++++ util.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/db/loose.py b/db/loose.py index c9ea3a038..a91f0d9d3 100644 --- a/db/loose.py +++ b/db/loose.py @@ -28,6 +28,7 @@ to_hex_sha, hex_to_bin, exists, + chmod, isdir, mkdir, rename, @@ -199,6 +200,10 @@ def store(self, istream): mkdir(obj_dir) # END handle destination directory rename(tmp_path, obj_path) + + # make sure its readable for all ! It started out as rw-- tmp file + # but needs to be rrr + chmod(obj_path, 0444) # END handle dry_run istream.sha = hexsha diff --git a/util.py b/util.py index f0bf3e60f..2d9838379 100644 --- a/util.py +++ b/util.py @@ -54,6 +54,7 @@ def unpack_from(fmt, data, offset=0): # os shortcuts exists = os.path.exists mkdir = os.mkdir +chmod = os.chmod isdir = os.path.isdir rename = os.rename dirname = os.path.dirname @@ -291,6 +292,9 @@ def _end_writing(self, successful=True): # END remove if exists # END win32 special handling os.rename(lockfile, self._filepath) + + # assure others can at least read the file - the tmpfile left it at rw-- + chmod(self._filepath, 0444) else: # just delete the file so far, we failed os.remove(lockfile) From e3d5ad195d9dfa46af3d931f9769e965e337daf7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 25 Jun 2010 15:29:50 +0200 Subject: [PATCH 0040/1392] CompoundDB: implemented simple dict base first-level cache which really helps to improve performance in real-world applications, which need to quickly determine whether objects are in or out for instance, as it happens during index_to_tree conversion Added separate pack streaming test which shows only a throughput of 250 streams / s in a densely packed pack, and about 3.5 MiB of data throughput. Performance tests show that half the time is spent in collecting the numerous deltas, the other one in decompressing and applying them Fixed broken performance tests --- db/base.py | 14 ++++++++++- db/git.py | 5 ++-- db/pack.py | 2 +- db/ref.py | 4 ++- exc.py | 7 +++++- test/performance/test_pack.py | 6 +++-- test/performance/test_pack_streaming.py | 33 +++++++++++++++++++++++++ test/performance/test_stream.py | 1 + 8 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 test/performance/test_pack_streaming.py diff --git a/db/base.py b/db/base.py index 35c20b7e1..0e81f0364 100644 --- a/db/base.py +++ b/db/base.py @@ -183,10 +183,13 @@ class CompoundDB(ObjectDBR, LazyMixin, CachingDB): Databases are stored in the lazy-loaded _dbs attribute. Define _set_cache_ to update it with your databases""" - def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() + elif attr == '_db_cache': + self._db_cache = dict() + else: + super(CompoundDB, self)._set_cache_(attr) def _db_query(self, sha): """:return: database containing the given 20 or 40 byte sha @@ -194,8 +197,15 @@ def _db_query(self, sha): # most databases use binary representations, prevent converting # it everytime a database is being queried sha = to_bin_sha(sha) + try: + return self._db_cache[sha] + except KeyError: + pass + # END first level cache + for db in self._dbs: if db.has_object(sha): + self._db_cache[sha] = db return db # END for each database raise BadObject(sha) @@ -232,6 +242,8 @@ def databases(self): return tuple(self._dbs) def update_cache(self, force=False): + # something might have changed, clear everything + self._db_cache.clear() stat = False for db in self._dbs: if isinstance(db, CachingDB): diff --git a/db/git.py b/db/git.py index ad9a613b3..a9298df05 100644 --- a/db/git.py +++ b/db/git.py @@ -57,8 +57,9 @@ def _set_cache_(self, attr): # finally set the value self._loose_db = loose_db - - # END handle dbs + else: + super(GitDB, self)._set_cache_(attr) + # END handle attrs #{ ObjectDBW interface diff --git a/db/pack.py b/db/pack.py index af6f7ffd6..a78c4a8a0 100644 --- a/db/pack.py +++ b/db/pack.py @@ -46,7 +46,7 @@ def __init__(self, root_path): def _set_cache_(self, attr): if attr == '_entities': self._entities = list() - self.update_cache() + self.update_cache(force=True) # END handle entities initialization def _sort_entities(self): diff --git a/db/ref.py b/db/ref.py index 3a4813979..c149c03d0 100644 --- a/db/ref.py +++ b/db/ref.py @@ -21,7 +21,9 @@ def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() self._update_dbs_from_ref_file() - # END handle dbs + else: + super(ReferenceDB, self)._set_cache_(attr) + # END handle attrs def _update_dbs_from_ref_file(self): dbcls = self.ObjectDBCls diff --git a/exc.py b/exc.py index 482726e3b..037ac3855 100644 --- a/exc.py +++ b/exc.py @@ -1,4 +1,5 @@ """Module with common exceptions""" +from util import to_hex_sha class ODBError(Exception): """All errors thrown by the object database""" @@ -7,7 +8,11 @@ class InvalidDBRoot(ODBError): """Thrown if an object database cannot be initialized at the given path""" class BadObject(ODBError): - """The object with the given SHA does not exist""" + """The object with the given SHA does not exist. Instantiate with the + failed sha""" + + def __str__(self): + return "BadObject: %s" % to_hex_sha(self.args[0]) class BadObjectType(ODBError): """The object had an unsupported type""" diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index 8046c1f83..66101a35f 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -72,8 +72,10 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) - - print >> sys.stderr, "Endurance run: verify streaming of %i objects (crc and sha)" % ns + def _disabled_test_correctness(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + # disabled for now as it used to work perfectly, checking big repositories takes a long time + print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" for crc in range(2): count = 0 st = time() diff --git a/test/performance/test_pack_streaming.py b/test/performance/test_pack_streaming.py new file mode 100644 index 000000000..4d47cdfcc --- /dev/null +++ b/test/performance/test_pack_streaming.py @@ -0,0 +1,33 @@ +"""Specific test for pack streams only""" +from lib import ( + TestBigRepoR + ) + +from gitdb.db.pack import PackedDB + +import os +import sys +from time import time + +class TestPackStreamingPerformance(TestBigRepoR): + + def test_stream_reading(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + # streaming only, meant for --with-profile runs + ni = 5000 + count = 0 + pdb_stream = pdb.stream + total_size = 0 + st = time() + for sha in pdb.sha_iter(): + if count == ni: + break + stream = pdb_stream(sha) + stream.read() + total_size += stream.size + count += 1 + elapsed = time() - st + total_kib = total_size / 1000 + print >> sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed) + diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 5de463ee2..79fa9bc09 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -1,6 +1,7 @@ """Performance data streaming performance""" from lib import TestBigRepoR from gitdb.db import * +from gitdb.base import * from gitdb.stream import * from gitdb.util import pool from gitdb.typ import str_blob_type From 9e313a4773c97425d5c52be34ee21cbe405ddb84 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 25 Jun 2010 16:39:25 +0200 Subject: [PATCH 0041/1392] gitdb now uses 20 byte shas internally only, reducing the need to convert shas around all the time, saving previous function calls, and memory after all --- base.py | 43 ++++++++++++++------------------- db/base.py | 17 ++++++------- db/loose.py | 18 +++++++------- db/mem.py | 6 ++--- db/pack.py | 6 +---- test/db/lib.py | 28 ++++++++++----------- test/db/test_git.py | 3 ++- test/db/test_ref.py | 15 +++++++----- test/performance/test_pack.py | 2 +- test/performance/test_stream.py | 13 ++++++---- test/test_base.py | 12 ++++----- test/test_pack.py | 14 +++++------ util.py | 1 + 13 files changed, 85 insertions(+), 93 deletions(-) diff --git a/base.py b/base.py index 0f4e63176..938a09242 100644 --- a/base.py +++ b/base.py @@ -1,7 +1,6 @@ """Module with basic data structures - they are designed to be lightweight and fast""" from util import ( - to_hex_sha, - to_bin_sha, + bin_to_hex, zlib ) @@ -17,13 +16,13 @@ #{ ODB Bases class OInfo(tuple): - """Carries information about an object in an ODB, provdiing information - about the sha of the object, the type_string as well as the uncompressed size + """Carries information about an object in an ODB, provding information + about the binary sha of the object, the type_string as well as the uncompressed size in bytes. It can be accessed using tuple notation and using attribute access notation:: - assert dbi[0] == dbi.sha + assert dbi[0] == dbi.binsha assert dbi[1] == dbi.type assert dbi[2] == dbi.size @@ -38,18 +37,14 @@ def __init__(self, *args): #{ Interface @property - def sha(self): + def binsha(self): + """:return: our sha as binary, 20 bytes""" return self[0] - + @property def hexsha(self): """:return: our sha, hex encoded, 40 bytes""" - return to_hex_sha(self[0]) - - @property - def binsha(self): - """:return: our sha as binary, 20 bytes""" - return to_bin_sha(self[0]) + return bin_to_hex(self[0]) @property def type(self): @@ -197,16 +192,10 @@ def __init__(self, type, size, stream, sha=None): list.__init__(self, (sha, type, size, stream, None)) #{ Interface - @property def hexsha(self): """:return: our sha, hex encoded, 40 bytes""" - return to_hex_sha(self[0]) - - @property - def binsha(self): - """:return: our sha as binary, 20 bytes""" - return to_bin_sha(self[0]) + return bin_to_hex(self[0]) def _error(self): """:return: the error that occurred when processing the stream, or None""" @@ -231,13 +220,13 @@ def read(self, size=-1): #{ interface - def _set_sha(self, sha): - self[0] = sha + def _set_binsha(self, binsha): + self[0] = binsha - def _sha(self): + def _binsha(self): return self[0] - sha = property(_sha, _set_sha) + binsha = property(_binsha, _set_binsha) def _type(self): @@ -280,9 +269,13 @@ def __init__(self, sha, exc): tuple.__init__(self, (sha, exc)) @property - def sha(self): + def binsha(self): return self[0] + @property + def hexsha(self): + return bin_to_hex(self[0]) + @property def error(self): """:return: exception instance explaining the failure""" diff --git a/db/base.py b/db/base.py index 0e81f0364..c687166f7 100644 --- a/db/base.py +++ b/db/base.py @@ -2,8 +2,7 @@ from gitdb.util import ( pool, join, - LazyMixin, - to_bin_sha + LazyMixin ) from gitdb.exc import BadObject @@ -20,8 +19,7 @@ class ObjectDBR(object): """Defines an interface for object database lookup. - Objects are identified either by hex-sha (40 bytes) or - by sha (20 bytes)""" + Objects are identified either by their 20 byte bin sha""" def __contains__(self, sha): return self.has_obj @@ -29,14 +27,14 @@ def __contains__(self, sha): #{ Query Interface def has_object(self, sha): """ - :return: True if the object identified by the given 40 byte hexsha or 20 bytes + :return: True if the object identified by the given 20 bytes binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") def has_object_async(self, reader): """Return a reader yielding information about the membership of objects as identified by shas - :param reader: Reader yielding 20 byte or 40 byte shas. + :param reader: Reader yielding 20 byte shas. :return: async.Reader yielding tuples of (sha, bool) pairs which indicate whether the given sha exists in the database or not""" task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) @@ -44,7 +42,7 @@ def has_object_async(self, reader): def info(self, sha): """ :return: OInfo instance - :param sha: 40 bytes hexsha or 20 bytes binary sha + :param sha: bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") @@ -57,7 +55,7 @@ def info_async(self, reader): def stream(self, sha): """:return: OStream instance - :param sha: 40 bytes hexsha or 20 bytes binary sha + :param sha: 20 bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") @@ -192,11 +190,10 @@ def _set_cache_(self, attr): super(CompoundDB, self)._set_cache_(attr) def _db_query(self, sha): - """:return: database containing the given 20 or 40 byte sha + """:return: database containing the given 20 byte sha :raise BadObject:""" # most databases use binary representations, prevent converting # it everytime a database is being queried - sha = to_bin_sha(sha) try: return self._db_cache[sha] except KeyError: diff --git a/db/loose.py b/db/loose.py index a91f0d9d3..7afc5bf1b 100644 --- a/db/loose.py +++ b/db/loose.py @@ -25,8 +25,8 @@ from gitdb.util import ( file_contents_ro_filepath, ENOENT, - to_hex_sha, hex_to_bin, + bin_to_hex, exists, chmod, isdir, @@ -100,7 +100,7 @@ def _map_loose_object(self, sha): """ :return: memory map of that file to allow random read access :raise BadObject: if object could not be located""" - db_path = self.db_path(self.object_path(to_hex_sha(sha))) + db_path = self.db_path(self.object_path(bin_to_hex(sha))) try: return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) except OSError,e: @@ -109,11 +109,11 @@ def _map_loose_object(self, sha): try: return file_contents_ro_filepath(db_path) except OSError: - raise BadObject(to_hex_sha(sha)) + raise BadObject(sha) # didn't work because of our flag, don't try it again self._fd_open_flags = 0 else: - raise BadObject(to_hex_sha(sha)) + raise BadObject(sha) # END handle error # END exception handling try: @@ -144,7 +144,7 @@ def stream(self, sha): def has_object(self, sha): try: - self.readable_db_object_path(to_hex_sha(sha)) + self.readable_db_object_path(bin_to_hex(sha)) return True except BadObject: return False @@ -158,7 +158,7 @@ def store(self, istream): # open a tmp file to write the data to fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - if istream.sha is None: + if istream.binsha is None: writer = FDCompressedSha1Writer(fd) else: writer = FDStream(fd) @@ -167,7 +167,7 @@ def store(self, istream): try: try: - if istream.sha is not None: + if istream.binsha is not None: # copy as much as possible, the actual uncompressed item size might # be smaller than the compressed version stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) @@ -187,7 +187,7 @@ def store(self, istream): # END assure tmpfile removal on error hexsha = None - if istream.sha: + if istream.binsha: hexsha = istream.hexsha else: hexsha = writer.sha(as_hex=True) @@ -206,7 +206,7 @@ def store(self, istream): chmod(obj_path, 0444) # END handle dry_run - istream.sha = hexsha + istream.binsha = hex_to_bin(hexsha) return istream def sha_iter(self): diff --git a/db/mem.py b/db/mem.py index 9e3d3972d..f361ab801 100644 --- a/db/mem.py +++ b/db/mem.py @@ -10,7 +10,6 @@ IStream, ) -from gitdb.util import to_bin_sha from gitdb.exc import ( BadObject, UnsupportedOperation @@ -54,7 +53,7 @@ def store(self, istream): # don't provide a size, the stream is written in object format, hence the # header needs decompression decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) - self._cache[istream.binsha] = OStream(istream.sha, istream.type, istream.size, decomp_stream) + self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) return istream @@ -62,14 +61,13 @@ def store_async(self, reader): raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") def has_object(self, sha): - return to_bin_sha(sha) in self._cache + return sha in self._cache def info(self, sha): # we always return streams, which are infos as well return self.stream(sha) def stream(self, sha): - sha = to_bin_sha(sha) try: ostream = self._cache[sha] # rewind stream for the next one to read diff --git a/db/pack.py b/db/pack.py index a78c4a8a0..1a1c390ea 100644 --- a/db/pack.py +++ b/db/pack.py @@ -5,10 +5,7 @@ CachingDB ) -from gitdb.util import ( - to_bin_sha, - LazyMixin - ) +from gitdb.util import LazyMixin from gitdb.exc import ( BadObject, @@ -65,7 +62,6 @@ def _pack_info(self, sha): self._sort_entities() # END update sorting - sha = to_bin_sha(sha) for item in self._entities: index = item[2](sha) if index is not None: diff --git a/test/db/lib.py b/test/db/lib.py index b22f9a2b8..0080d919e 100644 --- a/test/db/lib.py +++ b/test/db/lib.py @@ -41,15 +41,15 @@ def _assert_object_writing_simple(self, db): istream = IStream(str_blob_type, len(data), StringIO(data)) new_istream = db.store(istream) assert new_istream is istream - assert db.has_object(istream.sha) + assert db.has_object(istream.binsha) - info = db.info(istream.sha) + info = db.info(istream.binsha) assert isinstance(info, OInfo) assert info.type == istream.type and info.size == istream.size - stream = db.stream(istream.sha) + stream = db.stream(istream.binsha) assert isinstance(stream, OStream) - assert stream.sha == info.sha and stream.type == info.type + assert stream.binsha == info.binsha and stream.type == info.type assert stream.read() == data # END for each item @@ -80,10 +80,10 @@ def _assert_object_writing(self, db): # store returns same istream instance, with new sha set my_istream = db.store(istream) - sha = istream.sha + sha = istream.binsha assert my_istream is istream assert db.has_object(sha) != dry_run - assert len(sha) == 40 # for now we require 40 byte shas as default + assert len(sha) == 20 # verify data - the slow way, we want to run code if not dry_run: @@ -107,12 +107,12 @@ def _assert_object_writing(self, db): # identical to what we fed in ostream.seek(0) istream.stream = ostream - assert istream.sha is not None - prev_sha = istream.sha + assert istream.binsha is not None + prev_sha = istream.binsha db.set_ostream(ZippedStoreShaWriter()) db.store(istream) - assert istream.sha == prev_sha + assert istream.binsha == prev_sha new_ostream = db.ostream() # note: only works as long our store write uses the same compression @@ -143,12 +143,12 @@ def istream_generator(offset=0, ni=ni): for stream in istreams: assert stream.error is None - assert len(stream.sha) == 40 + assert len(stream.binsha) == 20 assert isinstance(stream, IStream) # END assert each stream # test has-object-async - we must have all previously added ones - reader = IteratorReader( istream.sha for istream in istreams ) + reader = IteratorReader( istream.binsha for istream in istreams ) hasobject_reader = db.has_object_async(reader) count = 0 for sha, has_object in hasobject_reader: @@ -158,7 +158,7 @@ def istream_generator(offset=0, ni=ni): assert count == ni # read the objects we have just written - reader = IteratorReader( istream.sha for istream in istreams ) + reader = IteratorReader( istream.binsha for istream in istreams ) ostream_reader = db.stream_async(reader) # read items individually to prevent hitting possible sys-limits @@ -171,7 +171,7 @@ def istream_generator(offset=0, ni=ni): assert count == ni # get info about our items - reader = IteratorReader( istream.sha for istream in istreams ) + reader = IteratorReader( istream.binsha for istream in istreams ) info_reader = db.info_async(reader) count = 0 @@ -186,7 +186,7 @@ def istream_generator(offset=0, ni=ni): # add 2500 items, and obtain their output streams nni = 2500 reader = IteratorReader(istream_generator(offset=ni, ni=nni)) - istream_to_sha = lambda istreams: [ istream.sha for istream in istreams ] + istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] istream_reader = db.store_async(reader) istream_reader.set_post_cb(istream_to_sha) diff --git a/test/db/test_git.py b/test/db/test_git.py index 4d463f1f7..779e3f15e 100644 --- a/test/db/test_git.py +++ b/test/db/test_git.py @@ -1,6 +1,7 @@ from lib import * from gitdb.db import GitDB from gitdb.base import OStream, OInfo +from gitdb.util import hex_to_bin class TestGitDB(TestDBBase): @@ -11,7 +12,7 @@ def test_reading(self): assert 1 < len(gdb.databases()) < 4 # access should be possible - gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976" + gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert isinstance(gdb.info(gitdb_sha), OInfo) assert isinstance(gdb.stream(gitdb_sha), OStream) assert gdb.size() > 200 diff --git a/test/db/test_ref.py b/test/db/test_ref.py index 68d9b8116..9df25cef6 100644 --- a/test/db/test_ref.py +++ b/test/db/test_ref.py @@ -1,6 +1,11 @@ from lib import * from gitdb.db import ReferenceDB - + +from gitdb.util import ( + NULL_BIN_SHA, + hex_to_bin + ) + import os class TestReferenceDB(TestDBBase): @@ -15,8 +20,7 @@ def make_alt_file(self, alt_path, alt_list): @with_rw_directory def test_writing(self, path): - null_sha_bin = '\0' * 20 - null_sha_hex = "0" * 40 + NULL_BIN_SHA = '\0' * 20 alt_path = os.path.join(path, 'alternates') rdb = ReferenceDB(alt_path) @@ -25,8 +29,7 @@ def test_writing(self, path): assert len(list(rdb.sha_iter())) == 0 # try empty, non-existing - assert not rdb.has_object(null_sha_hex) - assert not rdb.has_object(null_sha_bin) + assert not rdb.has_object(NULL_BIN_SHA) # setup alternate file @@ -37,7 +40,7 @@ def test_writing(self, path): assert len(rdb.databases()) == 1 # we should now find a default revision of ours - gitdb_sha = "5690fd0d3304f378754b23b098bd7cb5f4aa1976" + gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert rdb.has_object(gitdb_sha) # remove valid diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index 66101a35f..af468b0be 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -56,7 +56,7 @@ def test_pack_random_access(self): for sha in sha_list[:max_items]: pdb_fun(sha) elapsed = time() - st - print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f info/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) + print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) # END for each function # retrieve stream and read all diff --git a/test/performance/test_stream.py b/test/performance/test_stream.py index 79fa9bc09..1afc1a1a0 100644 --- a/test/performance/test_stream.py +++ b/test/performance/test_stream.py @@ -3,7 +3,10 @@ from gitdb.db import * from gitdb.base import * from gitdb.stream import * -from gitdb.util import pool +from gitdb.util import ( + pool, + bin_to_hex + ) from gitdb.typ import str_blob_type from gitdb.fun import chunk_size @@ -73,10 +76,10 @@ def test_large_data_streaming(self, path): # writing - due to the compression it will seem faster than it is st = time() - sha = ldb.store(IStream('blob', size, stream)).sha + sha = ldb.store(IStream('blob', size, stream)).binsha elapsed_add = time() - st assert ldb.has_object(sha) - db_file = ldb.readable_db_object_path(sha) + db_file = ldb.readable_db_object_path(bin_to_hex(sha)) fsize_kib = os.path.getsize(db_file) / 1000 @@ -151,7 +154,7 @@ def istream_iter(): # chunk size is not important as the stream will not really be decompressed # until its read - istream_reader = IteratorReader(iter([ i.sha for i in istreams ])) + istream_reader = IteratorReader(iter([ i.binsha for i in istreams ])) ostream_reader = ldb.stream_async(istream_reader) chunk_task = TestStreamReader(ostream_reader, "chunker", None) @@ -172,7 +175,7 @@ def istream_iter(): istream_reader = ldb.store_async(reader) istream_reader.task().max_chunksize = 1 - istream_to_sha = lambda items: [ i.sha for i in items ] + istream_to_sha = lambda items: [ i.binsha for i in items ] istream_reader.set_post_cb(istream_to_sha) ostream_reader = ldb.stream_async(istream_reader) diff --git a/test/test_base.py b/test/test_base.py index 8d8bcc944..740e50bcd 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -7,7 +7,7 @@ from gitdb import * from gitdb.util import ( - NULL_HEX_SHA + NULL_BIN_SHA ) from gitdb.typ import ( @@ -19,12 +19,12 @@ class TestBaseTypes(TestBase): def test_streams(self): # test info - sha = NULL_HEX_SHA + sha = NULL_BIN_SHA s = 20 blob_id = 3 info = OInfo(sha, str_blob_type, s) - assert info.sha == sha + assert info.binsha == sha assert info.type == str_blob_type assert info.type_id == blob_id assert info.size == s @@ -72,9 +72,9 @@ def test_streams(self): # test istream istream = IStream(str_blob_type, s, stream) - assert istream.sha == None - istream.sha = sha - assert istream.sha == sha + assert istream.binsha == None + istream.binsha = sha + assert istream.binsha == sha assert len(istream.binsha) == 20 assert len(istream.hexsha) == 40 diff --git a/test/test_pack.py b/test/test_pack.py index 6821097ad..eaa2d38eb 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -134,8 +134,8 @@ def test_pack_entity(self): count = 0 for info, stream in izip(entity.info_iter(), entity.stream_iter()): count += 1 - assert info.sha == stream.sha - assert len(info.sha) == 20 + assert info.binsha == stream.binsha + assert len(info.binsha) == 20 assert info.type_id == stream.type_id assert info.size == stream.size @@ -143,17 +143,17 @@ def test_pack_entity(self): assert not info.type_id in delta_types # try all calls - assert len(entity.collect_streams(info.sha)) - assert isinstance(entity.info(info.sha), OInfo) - assert isinstance(entity.stream(info.sha), OStream) + assert len(entity.collect_streams(info.binsha)) + assert isinstance(entity.info(info.binsha), OInfo) + assert isinstance(entity.stream(info.binsha), OStream) # verify the stream try: - assert entity.is_valid_stream(info.sha, use_crc=True) + assert entity.is_valid_stream(info.binsha, use_crc=True) except UnsupportedOperation: pass # END ignore version issues - assert entity.is_valid_stream(info.sha, use_crc=False) + assert entity.is_valid_stream(info.binsha, use_crc=False) # END for each info, stream tuple assert count == size diff --git a/util.py b/util.py index 2d9838379..4d6f5b1e2 100644 --- a/util.py +++ b/util.py @@ -66,6 +66,7 @@ def unpack_from(fmt, data, offset=0): # constants NULL_HEX_SHA = "0"*40 +NULL_BIN_SHA = "\0"*20 #} END Aliases From c265c97f9130d2225b923b427736796c0a0d957c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 29 Jun 2010 10:56:50 +0200 Subject: [PATCH 0042/1392] Fixed critical bug that could cause single bytes not to be returned in reads that should read all --- stream.py | 4 +--- .../7b/b839852ed5e3a069966281bb08d50012fb309b | Bin 0 -> 446 bytes test/test_stream.py | 15 ++++++++++++--- util.py | 3 ++- 4 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b diff --git a/stream.py b/stream.py index 3f5d0373a..010102bdd 100644 --- a/stream.py +++ b/stream.py @@ -249,9 +249,7 @@ def read(self, size=-1): # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) - dcompdat = self._zip.decompress(indata, size) - # update the amount of compressed bytes read # We feed possibly overlapping chunks, which is why the unconsumed tail # has to be taken into consideration, as well as the unused data @@ -269,7 +267,7 @@ def read(self, size=-1): # Note: dcompdat can be empty even though we still appear to have bytes # to read, if we are called by compressed_bytes_read - it manipulates # us to empty the stream - if dcompdat and len(dcompdat) < size and self._br < self._s: + if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s: dcompdat += self.read(size-len(dcompdat)) # END handle special case return dcompdat diff --git a/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b b/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b new file mode 100644 index 0000000000000000000000000000000000000000..021c2db3456031e1d2ab0bf73701fd07e1c3f18e GIT binary patch literal 446 zcmV;v0YUzF0V^p=O;s>8GGZ_^FfcPQQP4}zEJ-XWDauSLElDkA*k$BmEOsS@@9cup z94Ax`K0BwU`xdG)kwHqqdSOS!%y&8Fc7K0ZetX8c%q|BenU%5@~ z(-?H(E^K=G^sI4tWL0NrNXl|<-X9uJ$(;P;_{5YH25$lN^J%6=Iy|xceG!-L+2w=@ zF(E0*%}-&FY^mnnsdq=;+uk^0*18Z;q%))7P|&5i(pk;rewb_Um*b^Tbx?LaCci_=A6&R?hE*8 ztJ*Xl-{zzfCJ8maG_NQ%C$S_oB_0%@@wthac?_w(_A_#sd|uhiWEPKR`LO@V34K`5 opeZk|%uB|nyow>rIet2~&OwIvJHJh8HJG&J%aR?}0KQhvi8Wi_>i_@% literal 0 HcmV?d00001 diff --git a/test/test_stream.py b/test/test_stream.py index 859920719..dd65a1782 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -4,12 +4,14 @@ DummyStream, Sha1Writer, make_bytes, - make_object + make_object, + fixture_path ) from gitdb import * from gitdb.util import ( - NULL_HEX_SHA + NULL_HEX_SHA, + hex_to_bin ) from gitdb.util import zlib @@ -135,4 +137,11 @@ def test_compressed_writer(self): os.remove(path) # END for each os - + def test_decompress_reader_special_case(self): + odb = LooseObjectDB(fixture_path('objects')) + ostream = odb.stream(hex_to_bin('7bb839852ed5e3a069966281bb08d50012fb309b')) + + # if there is a bug, we will be missing one byte exactly ! + data = ostream.read() + assert len(data) == ostream.size + diff --git a/util.py b/util.py index 4d6f5b1e2..b55f789c0 100644 --- a/util.py +++ b/util.py @@ -56,6 +56,7 @@ def unpack_from(fmt, data, offset=0): mkdir = os.mkdir chmod = os.chmod isdir = os.path.isdir +isfile = os.path.isfile rename = os.rename dirname = os.path.dirname basename = os.path.basename @@ -288,7 +289,7 @@ def _end_writing(self, successful=True): if self._write and successful: # on windows, rename does not silently overwrite the existing one if sys.platform == "win32": - if os.path.isfile(self._filepath): + if isfile(self._filepath): os.remove(self._filepath) # END remove if exists # END win32 special handling From 155b62a9af0aa7677078331e111d0f7aa6eb4afc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 30 Jun 2010 00:05:38 +0200 Subject: [PATCH 0043/1392] Added empty version of gitdb documentation --- doc/Makefile | 89 ++++++++++++++++++++ doc/source/conf.py | 194 +++++++++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 20 +++++ 3 files changed, 303 insertions(+) create mode 100644 doc/Makefile create mode 100644 doc/source/conf.py create mode 100644 doc/source/index.rst diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 000000000..b10926ae2 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/GitDB.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/GitDB.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 000000000..58b8791a1 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +# +# GitDB documentation build configuration file, created by +# sphinx-quickstart on Wed Jun 30 00:01:32 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'GitDB' +copyright = u'2010, Sebastian Thiel' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '1.0' +# The full version, including alpha/beta/rc tags. +release = '1.0.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['.static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'GitDBdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'GitDB.tex', u'GitDB Documentation', + u'Sebastian Thiel', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 000000000..71ead1fc3 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,20 @@ +.. GitDB documentation master file, created by + sphinx-quickstart on Wed Jun 30 00:01:32 2010. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to GitDB's documentation! +================================= + +Contents: + +.. toctree:: + :maxdepth: 2 + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + From 78e21e74a12f0767f6011a78349af52555ad2f74 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 30 Jun 2010 11:22:05 +0200 Subject: [PATCH 0044/1392] Added auto-doc api reference and fixed plenty of docstrings --- db/base.py | 20 +++++-- db/pack.py | 4 +- doc/.gitignore | 1 + doc/source/api.rst | 113 ++++++++++++++++++++++++++++++++++++++++ doc/source/conf.py | 2 +- doc/source/index.rst | 4 ++ doc/source/intro.rst | 4 ++ doc/source/tutorial.rst | 4 ++ fun.py | 38 +++++++++----- pack.py | 39 +++++++++----- stream.py | 14 +++-- util.py | 18 ++++--- 12 files changed, 218 insertions(+), 43 deletions(-) create mode 100644 doc/.gitignore create mode 100644 doc/source/api.rst create mode 100644 doc/source/intro.rst create mode 100644 doc/source/tutorial.rst diff --git a/db/base.py b/db/base.py index c687166f7..02584c635 100644 --- a/db/base.py +++ b/db/base.py @@ -90,7 +90,9 @@ def __init__(self, *args, **kwargs): #{ Edit Interface def set_ostream(self, stream): - """Adjusts the stream to which all data should be sent when storing new objects + """ + Adjusts the stream to which all data should be sent when storing new objects + :param stream: if not None, the stream to use, if None the default stream will be used. :return: previously installed stream, or None if there was no override @@ -100,13 +102,16 @@ def set_ostream(self, stream): return cstream def ostream(self): - """:return: overridden output stream this instance will write to, or None + """ + :return: overridden output stream this instance will write to, or None if it will write to the default stream""" return self._ostream def store(self, istream): - """Create a new object in the database + """ + Create a new object in the database :return: the input istream object with its sha set to its corresponding value + :param istream: IStream compatible instance. If its sha is already set to a value, the object will just be stored in the our database format, in which case the input stream is expected to be in object format ( header + contents ). @@ -114,16 +119,19 @@ def store(self, istream): raise NotImplementedError("To be implemented in subclass") def store_async(self, reader): - """Create multiple new objects in the database asynchronously. The method will + """ + Create multiple new objects in the database asynchronously. The method will return right away, returning an output channel which receives the results as they are computed. :return: Channel yielding your IStream which served as input, in any order. The IStreams sha will be set to the sha it received during the process, or its error attribute will be set to the exception informing about the error. + :param reader: async.Reader yielding IStream instances. The same instances will be used in the output channel as were received in by the Reader. + :note:As some ODB implementations implement this operation atomic, they might abort the whole operation if one item could not be processed. Hence check how many items have actually been produced.""" @@ -167,8 +175,10 @@ class CachingDB(object): #{ Interface def update_cache(self, force=False): - """Call this method if the underlying data changed to trigger an update + """ + Call this method if the underlying data changed to trigger an update of the internal caching structures. + :param force: if True, the update must be performed. Otherwise the implementation may decide not to perform an update if it thinks nothing has changed. :return: True if an update was performed as something change indeed""" diff --git a/db/pack.py b/db/pack.py index 1a1c390ea..022efe0a2 100644 --- a/db/pack.py +++ b/db/pack.py @@ -128,8 +128,10 @@ def store_async(self, reader): #{ Interface def update_cache(self, force=False): - """Update our cache with the acutally existing packs on disk. Add new ones, + """ + Update our cache with the acutally existing packs on disk. Add new ones, and remove deleted ones. We keep the unchanged ones + :param force: If True, the cache will be updated even though the directory does not appear to have changed according to its modification timestamp. :return: True if the packs have been updated so there is new information, diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 000000000..567609b12 --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/doc/source/api.rst b/doc/source/api.rst new file mode 100644 index 000000000..417671da9 --- /dev/null +++ b/doc/source/api.rst @@ -0,0 +1,113 @@ +.. _api_reference_toplevel: + +############# +API Reference +############# + +**************** +Database.Base +**************** + +.. automodule:: gitdb.db.base + :members: + :undoc-members: + +**************** +Database.Git +**************** + +.. automodule:: gitdb.db.git + :members: + :undoc-members: + +**************** +Database.Loose +**************** + +.. automodule:: gitdb.db.loose + :members: + :undoc-members: + +**************** +Database.Memory +**************** + +.. automodule:: gitdb.db.mem + :members: + :undoc-members: + +**************** +Database.Pack +**************** + +.. automodule:: gitdb.db.pack + :members: + :undoc-members: + +****************** +Database.Reference +****************** + +.. automodule:: gitdb.db.ref + :members: + :undoc-members: + +************ +Base +************ + +.. automodule:: gitdb.base + :members: + :undoc-members: + +************ +Functions +************ + +.. automodule:: gitdb.fun + :members: + :undoc-members: + +************ +Pack +************ + +.. automodule:: gitdb.pack + :members: + :undoc-members: + +************ +Streams +************ + +.. automodule:: gitdb.stream + :members: + :undoc-members: + +************ +Types +************ + +.. automodule:: gitdb.typ + :members: + :undoc-members: + + +************ +Utilities +************ + +.. automodule:: gitdb.util + :members: + :undoc-members: + + + + + + + + + + + diff --git a/doc/source/conf.py b/doc/source/conf.py index 58b8791a1..d8aadabef 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,7 +16,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.append(os.path.abspath('.')) +sys.path.append(os.path.abspath('../../../')) # -- General configuration ----------------------------------------------------- diff --git a/doc/source/index.rst b/doc/source/index.rst index 71ead1fc3..409c78e99 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -10,6 +10,10 @@ Contents: .. toctree:: :maxdepth: 2 + + intro + tutorial + api Indices and tables ================== diff --git a/doc/source/intro.rst b/doc/source/intro.rst new file mode 100644 index 000000000..9565e766e --- /dev/null +++ b/doc/source/intro.rst @@ -0,0 +1,4 @@ +######## +Overview +######## + diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst new file mode 100644 index 000000000..b23a17703 --- /dev/null +++ b/doc/source/tutorial.rst @@ -0,0 +1,4 @@ + +######## +Tutorial +######## diff --git a/fun.py b/fun.py index d7e43717c..ccd8c0fc5 100644 --- a/fun.py +++ b/fun.py @@ -39,20 +39,22 @@ # used when dealing with larger streams chunk_size = 1000*mmap.PAGESIZE -__all__ = ('is_loose_object', 'loose_object_header_info', 'object_header_info', - 'write_object' ) +__all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data' ) #{ Routines def is_loose_object(m): - """:return: True the file contained in memory map m appears to be a loose object. - Only the first two bytes are needed""" + """ + :return: True the file contained in memory map m appears to be a loose object. + Only the first two bytes are needed""" b0, b1 = map(ord, m[:2]) word = (b0 << 8) + b1 return b0 == 0x78 and (word % 31) == 0 def loose_object_header_info(m): - """:return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + """ + :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" decompress_size = 8192 # is used in cgit as well @@ -61,9 +63,10 @@ def loose_object_header_info(m): return type_name, int(size) def pack_object_header_info(data): - """:return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) - The type_id should be interpreted according to the ``type_id_to_type_map`` map - The byte-offset specifies the start of the actual zlib compressed datastream + """ + :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) + The type_id should be interpreted according to the ``type_id_to_type_map`` map + The byte-offset specifies the start of the actual zlib compressed datastream :param m: random-access memory, like a string or memory map""" c = ord(data[0]) # first byte i = 1 # next char to read @@ -87,8 +90,9 @@ def pack_object_header_info(data): # END handle exceptions def msb_size(data, offset=0): - """:return: tuple(read_bytes, size) read the msb size from the given random - access data starting at the given byte offset""" + """ + :return: tuple(read_bytes, size) read the msb size from the given random + access data starting at the given byte offset""" size = 0 i = 0 l = len(data) @@ -107,12 +111,14 @@ def msb_size(data, offset=0): return i+offset, size def loose_object_header(type, size): - """:return: string representing the loose object header, which is immediately + """ + :return: string representing the loose object header, which is immediately followed by the content stream of size 'size'""" return "%s %i\0" % (type, size) def write_object(type, size, read, write, chunk_size=chunk_size): - """Write the object as identified by type, size and source_stream into the + """ + Write the object as identified by type, size and source_stream into the target_stream :param type: type string of the object @@ -131,8 +137,10 @@ def write_object(type, size, read, write, chunk_size=chunk_size): return tbw def stream_copy(read, write, size, chunk_size): - """Copy a stream up to size bytes using the provided read and write methods, + """ + Copy a stream up to size bytes using the provided read and write methods, in chunks of chunk_size + :note: its much like stream_copy utility, but operates just using methods""" dbw = 0 # num data bytes written @@ -156,8 +164,10 @@ def stream_copy(read, write, size, chunk_size): def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): - """Apply data from a delta buffer using a source buffer to the target file, + """ + Apply data from a delta buffer using a source buffer to the target file, which will be written to + :param src_buf: random access data from which the delta was created :param src_buf_size: size of the source buffer in bytes :param delta_buf_size: size fo the delta buffer in bytes diff --git a/pack.py b/pack.py index c66d6717e..ed873b269 100644 --- a/pack.py +++ b/pack.py @@ -383,8 +383,9 @@ def version(self): return self._version def data(self): - """:return: read-only data of this pack. It provides random access and usually - is a memory map""" + """ + :return: read-only data of this pack. It provides random access and usually + is a memory map""" return self._data def checksum(self): @@ -428,19 +429,22 @@ def collect_streams(self, offset): def info(self, offset): """Retrieve information about the object at the given file-absolute offset + :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._data, offset or self.first_object_offset, False)[1] def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information + :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._data, offset or self.first_object_offset, True)[1] def stream_iter(self, start_offset=0): - """:return: iterator yielding OPackStream compatible instances, allowing - to access the data in the pack directly. + """ + :return: iterator yielding OPackStream compatible instances, allowing + to access the data in the pack directly. :param start_offset: offset to the first object to iterate. If 0, iteration starts at the very first object in the pack. :note: Iterating a pack directly is costly as the datastream has to be decompressed @@ -555,6 +559,7 @@ def _object(self, sha, as_stream, index=-1): def info(self, sha): """Retrieve information about the object identified by the given sha + :param sha: 20 byte sha1 :raise BadObject: :return: OInfo instance, with 20 byte sha""" @@ -562,6 +567,7 @@ def info(self, sha): def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha + :param sha: 20 byte sha1 :raise BadObject: :return: OStream instance, with 20 byte sha""" @@ -589,15 +595,18 @@ def index(self): return self._index def is_valid_stream(self, sha, use_crc=False): - """Verify that the stream at the given sha is valid. - :param sha: 20 byte sha1 of the object whose stream to verify + """ + Verify that the stream at the given sha is valid. + :param use_crc: if True, the index' crc for the sha is used to determine - whether the compressed stream of the object is valid. If it is + :param sha: 20 byte sha1 of the object whose stream to verify + whether the compressed stream of the object is valid. If it is a delta, this only verifies that the delta's data is valid, not the data of the actual undeltified object, as it depends on more than just this stream. If False, the object will be decompressed and the sha generated. It must match the given sha + :return: True if the stream is valid :raise UnsupportedOperation: If the index is version 1 only :raise BadObject: sha was not found""" @@ -639,18 +648,22 @@ def is_valid_stream(self, sha, use_crc=False): return True def info_iter(self): - """:return: Iterator over all objects in this pack. The iterator yields + """ + :return: Iterator over all objects in this pack. The iterator yields OInfo instances""" return self._iter_objects(as_stream=False) def stream_iter(self): - """:return: iterator over all objects in this pack. The iterator yields - OStream instances""" + """ + :return: iterator over all objects in this pack. The iterator yields + OStream instances""" return self._iter_objects(as_stream=True) def collect_streams_at_offset(self, offset): - """As the version in the PackFile, but can resolve REF deltas within this pack + """ + As the version in the PackFile, but can resolve REF deltas within this pack For more info, see ``collect_streams`` + :param offset: offset into the pack file at which the object can be found""" streams = self._pack.collect_streams(offset) @@ -678,11 +691,13 @@ def collect_streams_at_offset(self, offset): return streams def collect_streams(self, sha): - """As ``PackFile.collect_streams``, but takes a sha instead of an offset. + """ + As ``PackFile.collect_streams``, but takes a sha instead of an offset. Additionally, ref_delta streams will be resolved within this pack. If this is not possible, the stream will be left alone, hence it is adivsed to check for unresolved ref-deltas and resolve them before attempting to construct a delta stream. + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect :return: list of streams, first being the actual object delta, the last being a possibly unresolved base object. diff --git a/stream.py b/stream.py index 010102bdd..675ccf27e 100644 --- a/stream.py +++ b/stream.py @@ -77,6 +77,7 @@ def __del__(self): def _parse_header_info(self): """If this stream contains object data, parse the header info and skip the stream to a point where each read will yield object content + :return: parsed type_string, size""" # read header maxb = 512 # should really be enough, cgit uses 8192 I believe @@ -105,6 +106,7 @@ def new(self, m, close_on_deletion=False): """Create a new DecompressMemMapReader instance for acting as a read-only stream This method parses the object header from m and returns the parsed type and size, as well as the created stream instance. + :param m: memory map on which to oparate. It must be object data ( header + contents ) :param close_on_deletion: if True, the memory map will be closed once we are being deleted""" @@ -117,8 +119,9 @@ def data(self): return self._m def compressed_bytes_read(self): - """:return: number of compressed bytes read. This includes the bytes it - took to decompress the header ( if there was one )""" + """ + :return: number of compressed bytes read. This includes the bytes it + took to decompress the header ( if there was one )""" # ABSTRACT: When decompressing a byte stream, it can be that the first # x bytes which were requested match the first x bytes in the loosely # compressed datastream. This is the worst-case assumption that the reader @@ -406,6 +409,7 @@ def read(self, count=0): def seek(self, offset, whence=os.SEEK_SET): """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" if offset != 0 or whence != os.SEEK_SET: raise ValueError("Can only seek to position 0") @@ -417,11 +421,14 @@ def seek(self, offset, whence=os.SEEK_SET): @classmethod def new(cls, stream_list): - """Convert the given list of streams into a stream which resolves deltas + """ + Convert the given list of streams into a stream which resolves deltas when reading from it. + :param stream_list: two or more stream objects, first stream is a Delta to the object that you want to resolve, followed by N additional delta streams. The list's last stream must be a non-delta stream. + :return: Non-Delta OPackStream object whose stream can be used to obtain the decompressed resolved data :raise ValueError: if the stream list cannot be handled""" @@ -526,6 +533,7 @@ def getvalue(self): class FDCompressedSha1Writer(Sha1Writer): """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor + :note: operates on raw file descriptors :note: for this to work, you have to use the close-method of this instance""" __slots__ = ("fd", "sha1", "zip") diff --git a/util.py b/util.py index b55f789c0..502ac94dd 100644 --- a/util.py +++ b/util.py @@ -154,25 +154,26 @@ def to_bin_sha(sha): class LazyMixin(object): """ - Base class providing an interface to lazily retrieve attribute values upon + Base class providing an interface to lazily retrieve attribute values upon first access. If slots are used, memory will only be reserved once the attribute - is actually accessed and retrieved the first time. All future accesses will + is actually accessed and retrieved the first time. All future accesses will return the cached value as stored in the Instance's dict or slot. """ + __slots__ = tuple() def __getattr__(self, attr): """ Whenever an attribute is requested that we do not know, we allow it to be created and set. Next time the same attribute is reqeusted, it is simply - returned from our dict/slots. - """ + returned from our dict/slots. """ self._set_cache_(attr) # will raise in case the cache was not created return object.__getattribute__(self, attr) def _set_cache_(self, attr): - """ This method should be overridden in the derived class. + """ + This method should be overridden in the derived class. It should check whether the attribute named by attr can be created and cached. Do nothing if you do not know the attribute or call your subclass @@ -183,7 +184,8 @@ def _set_cache_(self, attr): class LockedFD(object): - """This class facilitates a safe read and write operation to a file on disk. + """ + This class facilitates a safe read and write operation to a file on disk. If we write to 'file', we obtain a lock file at 'file.lock' and write to that instead. If we succeed, the lock file will be renamed to overwrite the original file. @@ -212,7 +214,9 @@ def _lockfilepath(self): return "%s.lock" % self._filepath def open(self, write=False, stream=False): - """Open the file descriptor for reading or writing, both in binary mode. + """ + Open the file descriptor for reading or writing, both in binary mode. + :param write: if True, the file descriptor will be opened for writing. Other wise it will be opened read-only. :param stream: if True, the file descriptor will be wrapped into a simple stream From 4cde3c046b71bf481418cd3a2a0f0bf5bc540a2b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 30 Jun 2010 21:52:12 +0200 Subject: [PATCH 0045/1392] Added a minimal documentation, including a quick usage guide --- doc/source/api.rst | 2 +- doc/source/conf.py | 11 ++-- doc/source/intro.rst | 25 +++++++++ doc/source/tutorial.rst | 116 ++++++++++++++++++++++++++++++++++++++-- test/test_example.py | 53 ++++++++++++++++++ 5 files changed, 197 insertions(+), 10 deletions(-) create mode 100644 test/test_example.py diff --git a/doc/source/api.rst b/doc/source/api.rst index 417671da9..9dea95764 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -1,4 +1,4 @@ -.. _api_reference_toplevel: +.. _api-label: ############# API Reference diff --git a/doc/source/conf.py b/doc/source/conf.py index d8aadabef..8e2585c2f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -45,9 +45,9 @@ # built documents. # # The short X.Y version. -version = '1.0' +version = '0.5' # The full version, including alpha/beta/rc tags. -release = '1.0.0' +release = '0.5.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -93,10 +93,9 @@ # Sphinx are currently 'default' and 'sphinxdoc'. html_theme = 'default' -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} +html_theme_options = { + "stickysidebar": "true" +} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 9565e766e..a51d8bff4 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -2,3 +2,28 @@ Overview ######## +The *GitDB* project implements interfaces to allow read and write access to git repositories. In its core lies the *db* package, which contains all database types necessary to read a complete git repository. These are the ``LooseObjectDB``, the ``PackedDB`` and the ``ReferenceDB`` which are combined into the ``GitDB`` to combine every aspect of the git database. + +For this to work, GitDB implements pack reading, as well as loose object reading and writing. Data is always encapsulated in streams, which allows huge files to be handled as well as small ones, usually only chunks of the stream are kept in memory for processing, never the whole stream at once. + +Interfaces are used to describe the API, making it easy to provide alternate implementations. + +================ +Installing GitDB +================ +Its easiest to install gitdb using the *easy_install* program, which is part of the `setuptools`_:: + + $ easy_install gitdb + +As the command will install gitdb in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: + + $ python setup.py install + +=============== +Getting Started +=============== +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + +.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index b23a17703..cfe3fb284 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -1,4 +1,114 @@ +.. _tutorial-label: -######## -Tutorial -######## +########### +Usage Guide +########### +This text briefly introduces you to the basic design decisions and accompanying types. + +****** +Design +****** +The *GitDB* project models a standard git object database and implements it in pure python. This means that data, being classified by one of four types, can can be stored in the database and will in future be referred to by the generated SHA1 key, which is a 20 byte string within python. + +*GitDB* implements *RW* access to loose objects, as well as *RO* access to packed objects. Compound Databases allow to combine multiple object databases into one. + +All data is read and written using streams, which effectively prevents more than a chunk of the data being kept in memory at once mostly [#]_. + +******* +Streams +******* +In order to assure the object database can handle objects of any size, a stream interface is used for data retrieval as well as to fill data into the database. + +Basic Stream Types +================== +There are two fundamentally different types of streams, **IStream**\ s and **OStream**\ s. IStreams are mutable and are used to provide data streams to the database to create new objects. + +OStreams are immutable and are used to read data from the database. The base of this type, **OInfo**, contains only type and size information of the queried object, but no stream, which is slightly faster to retrieve depending on the database. + +OStreams are tuples, IStreams are lists. Both, OInfo and OStream, have the same member ordering which allows quick conversion from one type to another. + +**************************** +Data Query and Data Addition +**************************** +Databases support query and/or addition of objects using simple interfaces. They are called **ObjectDBR** for read-only access, and **ObjectDBW** for write access to create new objects. + +Both have two sets of methods, one of which allows interacting with single objects, the other one allowing to handle a stream of objects simultaneously and asynchronously. + +Acquiring information about an object from a database is easy if you have a SHA1 to refer to the object:: + + + ldb = LooseObjectDB(fixture_path("../../.git/objects")) + + for sha1 in ldb.sha_iter(): + oinfo = ldb.info(sha1) + ostream = ldb.stream(sha1) + assert oinfo[:3] == ostream[:3] + + assert len(ostream.read()) == ostream.size + # END for each sha in database + +To store information, you prepare an *IStream* object with the required information. The provided stream will be read and converted into an object, and the respective 20 byte SHA1 identifier is stored in the IStream object:: + + data = "my data" + istream = IStream("blob", len(data), StringIO(data)) + + # the object does not yet have a sha + assert istream.binsha is None + ldb.store(istream) + # now the sha is set + assert len(istream.binsha) == 20 + assert ldb.has_object(istream.binsha) + +********************** +Asynchronous Operation +********************** +For each read or write method that allows a single-object to be handled, an *_async* version exists which reads items to be processed from a channel, and writes the operation's result into an output channel that is read by the caller or by other async methods, to support chaining. + +Using asynchronous operations is easy, but chaining multiple operations together to form a complex one would require you to read the docs of the *async* package. At the current time, due to the *GIL*, the *GitDB* can only achieve true concurrency during zlib compression and decompression if big objects, if the respective c modules where compiled in *async*. + +Asynchronous operations are scheduled by a *ThreadPool* which resides in the *gitdb.util* module:: + + from gitdb.util import pool + + # set the pool to use two threads + pool.set_size(2) + + # synchronize the mode of operation + pool.set_size(0) + + +Use async methods with readers, which supply items to be processed. The result is given through readers as well:: + + from async import IteratorReader + + # Create a reader from an iterator + reader = IteratorReader(ldb.sha_iter()) + + # get reader for object streams + info_reader = ldb.stream_async(reader) + + # read one + info = info_reader.read(1)[0] + + # read all the rest until depletion + ostreams = info_reader.read() + + + +********* +Databases +********* +A database implements different interfaces, one if which will always be the *ObjectDBR* interface to support reading of object information and streams. + +The *Loose Object Database* as well as the *Packed Object Database* are *File Databases*, hence they operate on a directory which contains files they can read. + +File databases implementing the *ObjectDBW* interface can also be forced to write their output into the specified stream, using the ``set_ostream`` method. This effectively allows you to redirect its output to anywhere you like. + +*Compound Databases* are not implementing their own access type, but instead combine multiple database implementations into one. Examples for this database type are the *Reference Database*, which reads object locations from a file, and the *GitDB* which combines loose, packed and referenced objects into one database interface. + +For more information about the individual database types, please see the :ref:`API Reference `, and the unittests for the respective types. + + +---- + +.. [#] When reading streams from packs, all deltas are currently applied and the result written into a memory map before the first byte is returned. Future versions of the delta-apply algorithm might improve on this. diff --git a/test/test_example.py b/test/test_example.py new file mode 100644 index 000000000..dc3d6230a --- /dev/null +++ b/test/test_example.py @@ -0,0 +1,53 @@ +"""Module with examples from the tutorial section of the docs""" +from lib import * +from gitdb import IStream +from gitdb.db import LooseObjectDB +from gitdb.util import pool + +from cStringIO import StringIO + +from async import IteratorReader + +class TestExamples(TestBase): + + def test_base(self): + ldb = LooseObjectDB(fixture_path("../../.git/objects")) + + for sha1 in ldb.sha_iter(): + oinfo = ldb.info(sha1) + ostream = ldb.stream(sha1) + assert oinfo[:3] == ostream[:3] + + assert len(ostream.read()) == ostream.size + assert ldb.has_object(oinfo.binsha) + # END for each sha in database + + data = "my data" + istream = IStream("blob", len(data), StringIO(data)) + + # the object does not yet have a sha + assert istream.binsha is None + ldb.store(istream) + # now the sha is set + assert len(istream.binsha) == 20 + assert ldb.has_object(istream.binsha) + + + # async operation + # Create a reader from an iterator + reader = IteratorReader(ldb.sha_iter()) + + # get reader for object streams + info_reader = ldb.stream_async(reader) + + # read one + info = info_reader.read(1)[0] + + # read all the rest until depletion + ostreams = info_reader.read() + + # set the pool to use two threads + pool.set_size(2) + + # synchronize the mode of operation + pool.set_size(0) From 7562fdd96ab995f6c25fc102ef40a285283c844e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jul 2010 13:30:14 +0200 Subject: [PATCH 0046/1392] added setup.py and Manifest to allow distribution and easy_install --- .gitignore | 3 +++ MANIFEST.in | 15 +++++++++++++++ README | 2 +- doc/source/intro.rst | 12 ++++++++++++ ext/async | 2 +- setup.py | 18 ++++++++++++++++++ 6 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 MANIFEST.in create mode 100755 setup.py diff --git a/.gitignore b/.gitignore index 1a9d961a7..1e097f6f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +MANIFEST +build/ +dist/ *.pyc *.o *.so diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..a01acc452 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,15 @@ +include VERSION +include LICENSE +include CHANGES +include AUTHORS +include README + +include _fun.c + +graft test + +global-exclude .git* +global-exclude *.pyc +global-exclude *.so +global-exclude *.dll +global-exclude *.o diff --git a/README b/README index a52d9f508..33a566d86 100644 --- a/README +++ b/README @@ -1,4 +1,4 @@ -GtDB +GitDB ===== GitDB allows you to access bare git repositories for reading and writing. It diff --git a/doc/source/intro.rst b/doc/source/intro.rst index a51d8bff4..4d675cbc4 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -26,4 +26,16 @@ Getting Started =============== It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. +================= +Source Repository +================= +The latest source can be cloned using git from one of the following locations: + + * git://gitorious.org/git-python/gitdb.git + * git://github.com/Byron/gitdb.git + +License Information +=================== +*GitDB* is licensed under the New BSD License. + .. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools diff --git a/ext/async b/ext/async index 796b5e94f..a18235276 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 796b5e94f19dfc36a3fb251468192373c76510b0 +Subproject commit a1823527631ffa8a2438c95096e71a741df7b61e diff --git a/setup.py b/setup.py new file mode 100755 index 000000000..b6c2c414b --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +from distutils.core import setup, Extension + +setup(name = "gitdb", + version = "0.5.0", + description = "Git Object Database", + author = "Sebastian Thiel", + author_email = "byronimo@gmail.com", + url = "http://gitorious.org/git-python/gitdb", + packages = ('gitdb', 'gitdb.db', 'gitdb.test', 'gitdb.test.db', 'gitdb.test.performance'), + package_data={'gitdb' : ['AUTHORS', 'README'], + 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, + package_dir = {'gitdb':''}, + ext_modules=[Extension('gitdb._fun', ['_fun.c'])], + license = "BSD License", + requires=('async (>=0.6.0)',), + long_description = """GitDB is a pure-Python git object database""" + ) From b76bac22bbb146a7a82318721147d7a5f831b7e9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jul 2010 18:23:53 +0200 Subject: [PATCH 0047/1392] Removed obsolete makefile, to compile the extension, use ./setup build_ext and copy _fun.so from the build directory into the root directory --- makefile | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 makefile diff --git a/makefile b/makefile deleted file mode 100644 index 390289625..000000000 --- a/makefile +++ /dev/null @@ -1,12 +0,0 @@ - -_fun.o: _fun.c - gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -I/usr/include/python2.6 -c $< -o $@ - -_fun.so: _fun.o - gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions $^ -o $@ - -all: _fun.so - -clean: - -rm *.so - -rm *.o From 6c8721a7d5d32e54bb4ffd3725ed23ac5d76a593 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 2 Jul 2010 16:22:40 +0200 Subject: [PATCH 0048/1392] Win32 compatability adjustments. One performance test fails during shutdown as python fails to delete a stream in time it seems, so the file cannot be deleted as it is still opened by someone. This is madness, raising the question how badly python truly fails to call the destructors of objects in order to allow them to release their resources --- db/loose.py | 16 ++++++++++++++-- test/lib.py | 6 ++++++ test/test_example.py | 3 +++ test/test_stream.py | 5 ++++- util.py | 8 ++++++-- 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/db/loose.py b/db/loose.py index 7afc5bf1b..86eb151fb 100644 --- a/db/loose.py +++ b/db/loose.py @@ -30,6 +30,8 @@ exists, chmod, isdir, + isfile, + remove, mkdir, rename, dirname, @@ -60,6 +62,12 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): # chunks in which data will be copied between streams stream_chunk_size = chunk_size + # On windows we need to keep it writable, otherwise it cannot be removed + # either + new_objects_mode = 0444 + if os.name == 'nt': + new_objects_mode = 0644 + def __init__(self, root_path): super(LooseObjectDB, self).__init__(root_path) @@ -199,11 +207,15 @@ def store(self, istream): if not isdir(obj_dir): mkdir(obj_dir) # END handle destination directory + # rename onto existing doesn't work on windows + if os.name == 'nt' and isfile(obj_path): + remove(obj_path) + # END handle win322 rename(tmp_path, obj_path) # make sure its readable for all ! It started out as rw-- tmp file - # but needs to be rrr - chmod(obj_path, 0444) + # but needs to be rwrr + chmod(obj_path, self.new_objects_mode) # END handle dry_run istream.binsha = hex_to_bin(hexsha) diff --git a/test/lib.py b/test/lib.py index 742aa7f5c..3fb87d547 100644 --- a/test/lib.py +++ b/test/lib.py @@ -19,6 +19,7 @@ import tempfile import shutil import os +import gc #{ Bases @@ -44,6 +45,11 @@ def wrapper(self): print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) raise finally: + # Need to collect here to be sure all handles have been closed. It appears + # a windows-only issue. In fact things should be deleted, as well as + # memory maps closed, once objects go out of scope. For some reason + # though this is not the case here unless we collect explicitly. + gc.collect() shutil.rmtree(path) # END handle exception # END wrapper diff --git a/test/test_example.py b/test/test_example.py index dc3d6230a..dc82436ec 100644 --- a/test/test_example.py +++ b/test/test_example.py @@ -21,6 +21,9 @@ def test_base(self): assert len(ostream.read()) == ostream.size assert ldb.has_object(oinfo.binsha) # END for each sha in database + # assure we close all files + del(ostream) + del(oinfo) data = "my data" istream = IStream("blob", len(data), StringIO(data)) diff --git a/test/test_stream.py b/test/test_stream.py index dd65a1782..948cbe766 100644 --- a/test/test_stream.py +++ b/test/test_stream.py @@ -19,6 +19,7 @@ str_blob_type ) +import time import tempfile import os @@ -125,12 +126,14 @@ def test_compressed_writer(self): # for now, just a single write, code doesn't care about chunking assert len(data) == ostream.write(data) ostream.close() + # its closed already self.failUnlessRaises(OSError, os.close, fd) # read everything back, compare to data we zip - fd = os.open(path, os.O_RDONLY) + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) written_data = os.read(fd, os.path.getsize(path)) + assert len(written_data) == os.path.getsize(path) os.close(fd) assert written_data == zlib.compress(data, 1) # best speed diff --git a/util.py b/util.py index 502ac94dd..6b2f91073 100644 --- a/util.py +++ b/util.py @@ -58,12 +58,14 @@ def unpack_from(fmt, data, offset=0): isdir = os.path.isdir isfile = os.path.isfile rename = os.rename +remove = os.remove dirname = os.path.dirname basename = os.path.basename join = os.path.join read = os.read write = os.write close = os.close +fsync = os.fsync # constants NULL_HEX_SHA = "0"*40 @@ -128,7 +130,7 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): :note: for now we don't try to use O_NOATIME directly as the right value needs to be shared per database in fact. It only makes a real difference for loose object databases anyway, and they use it with the help of the ``flags`` parameter""" - fd = os.open(filepath, os.O_RDONLY|flags) + fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) try: return file_contents_ro(fd, stream, allow_mmap) finally: @@ -300,7 +302,9 @@ def _end_writing(self, successful=True): os.rename(lockfile, self._filepath) # assure others can at least read the file - the tmpfile left it at rw-- - chmod(self._filepath, 0444) + # We may also write that file, on windows that boils down to a remove- + # protection as well + chmod(self._filepath, 0644) else: # just delete the file so far, we failed os.remove(lockfile) From 46bf4710e0f7184ac4875e8037de30b5081bfda2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jul 2010 00:34:41 +0200 Subject: [PATCH 0049/1392] PackEntity: fixed capital bug which would cause a None to be in place of the bin sha when querying infos or streams through an entity --- ext/async | 2 +- pack.py | 4 +++- test/test_pack.py | 8 ++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ext/async b/ext/async index a18235276..76f15fc4b 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit a1823527631ffa8a2438c95096e71a741df7b61e +Subproject commit 76f15fc4b3e3ccb0160d6c887181f29095d16f23 diff --git a/pack.py b/pack.py index ed873b269..5b13bcc56 100644 --- a/pack.py +++ b/pack.py @@ -516,6 +516,9 @@ def _object(self, sha, as_stream, index=-1): # its a little bit redundant here, but it needs to be efficient if index < 0: index = self._sha_to_index(sha) + if sha is None: + sha = self._index.sha(index) + # END assure sha is present ( in output ) offset = self._index.offset(index) type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset)) if as_stream: @@ -551,7 +554,6 @@ def _object(self, sha, as_stream, index=-1): # collect the streams to obtain the actual object type if streams[-1].type_id in delta_types: raise BadObject(sha, "Could not resolve delta object") - return OInfo(sha, streams[-1].type, target_size) # END handle stream diff --git a/test/test_pack.py b/test/test_pack.py index eaa2d38eb..445ed4a17 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -144,8 +144,12 @@ def test_pack_entity(self): # try all calls assert len(entity.collect_streams(info.binsha)) - assert isinstance(entity.info(info.binsha), OInfo) - assert isinstance(entity.stream(info.binsha), OStream) + oinfo = entity.info(info.binsha) + assert isinstance(oinfo, OInfo) + assert oinfo.binsha is not None + ostream = entity.stream(info.binsha) + assert isinstance(ostream, OStream) + assert ostream.binsha is not None # verify the stream try: From e750ce0be7d3cbf694c095f6519e2e34b2c3332c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jul 2010 10:40:35 +0200 Subject: [PATCH 0050/1392] Added method to obtain a full sha from a partial sha, for each of the databases. The IndexFile has a new method to retrieve an index from a partial sha, instead of from a full sha, to provide the required functionality --- db/git.py | 65 ++++++++++++++++++++++++++++++++++++++++++- db/loose.py | 20 ++++++++++++- db/pack.py | 22 +++++++++++++++ exc.py | 6 +++- pack.py | 48 +++++++++++++++++++++++++++++++- test/db/test_git.py | 13 +++++++-- test/db/test_loose.py | 14 +++++++++- test/db/test_pack.py | 24 ++++++++++++++++ test/test_pack.py | 6 ++++ 9 files changed, 211 insertions(+), 7 deletions(-) diff --git a/db/git.py b/db/git.py index a9298df05..62fed7fb6 100644 --- a/db/git.py +++ b/db/git.py @@ -9,11 +9,33 @@ from ref import ReferenceDB from gitdb.util import LazyMixin -from gitdb.exc import InvalidDBRoot +from gitdb.exc import ( + InvalidDBRoot, + BadObject, + AmbiguousObjectName + ) import os +from gitdb.util import hex_to_bin + __all__ = ('GitDB', ) + +def _databases_recursive(database, output): + """Fill output list with database from db, in order. Deals with Loose, Packed + and compound databases.""" + if isinstance(database, CompoundDB): + compounds = list() + dbs = database.databases() + output.extend(db for db in dbs if not isinstance(db, CompoundDB)) + for cdb in (db for db in dbs if isinstance(db, CompoundDB)): + _databases_recursive(cdb, output) + else: + output.append(database) + # END handle database type + + + class GitDB(FileDBBase, ObjectDBW, CompoundDB): """A git-style object database, which contains all objects in the 'objects' subdirectory""" @@ -71,4 +93,45 @@ def ostream(self): def set_ostream(self, ostream): return self._loose_db.set_ostream(ostream) + #} END objectdbw interface + + #{ Interface + + def partial_to_complete_sha_hex(self, partial_hexsha): + """ + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha + :param partial_hexsha: hexsha with less than 40 byte + :raise AmbiguousObjectName: """ + databases = list() + _databases_recursive(self, databases) + + if len(partial_hexsha) % 2 != 0: + partial_binsha = hex_to_bin(partial_hexsha + "0") + else: + partial_binsha = hex_to_bin(partial_hexsha) + # END assure successful binary conversion + + candidate = None + for db in databases: + full_bin_sha = None + try: + if isinstance(db, LooseObjectDB): + full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) + else: + full_bin_sha = db.partial_to_complete_sha(partial_binsha) + # END handle database type + except BadObject: + continue + # END ignore bad objects + if full_bin_sha: + if candidate and candidate != full_bin_sha: + raise AmbiguousObjectName(partial_hexsha) + candidate = full_bin_sha + # END handle candidate + # END for each db + if not candidate: + raise BadObject(partial_binsha) + return candidate + + #} END interface diff --git a/db/loose.py b/db/loose.py index 86eb151fb..521be44c2 100644 --- a/db/loose.py +++ b/db/loose.py @@ -7,7 +7,8 @@ from gitdb.exc import ( InvalidDBRoot, - BadObject, + BadObject, + AmbiguousObjectName ) from gitdb.stream import ( @@ -102,6 +103,23 @@ def readable_db_object_path(self, hexsha): # END handle cache raise BadObject(hexsha) + def partial_to_complete_sha_hex(self, partial_hexsha): + """:return: 20 byte binary sha1 string which matches the given name uniquely + :param name: hexadecimal partial name + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for binsha in self.sha_iter(): + if bin_to_hex(binsha).startswith(partial_hexsha): + # it can't ever find the same object twice + if candidate is not None: + raise AmbiguousObjectName(partial_hexsha) + candidate = binsha + # END for each object + if candidate is None: + raise BadObject(partial_hexsha) + return candidate + #} END interface def _map_loose_object(self, sha): diff --git a/db/pack.py b/db/pack.py index 022efe0a2..47d511378 100644 --- a/db/pack.py +++ b/db/pack.py @@ -10,6 +10,7 @@ from gitdb.exc import ( BadObject, UnsupportedOperation, + AmbiguousObjectName ) from gitdb.pack import PackEntity @@ -175,5 +176,26 @@ def update_cache(self, force=False): def entities(self): """:return: list of pack entities operated upon by this database""" return [ item[1] for item in self._entities ] + + def partial_to_complete_sha(self, partial_binsha): + """:return: 20 byte sha as inferred by the given partial binary sha + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for item in self._entities: + item_index = item[1].index().partial_sha_to_index(partial_binsha) + if item_index is not None: + sha = item[1].index().sha(item_index) + if candidate and candidate != sha: + raise AmbiguousObjectName(partial_binsha) + candidate = sha + # END handle full sha could be found + # END for each entity + + if candidate: + return candidate + + # still not found ? + raise BadObject(partial_binsha) #} END interface diff --git a/exc.py b/exc.py index 037ac3855..012cdbc6b 100644 --- a/exc.py +++ b/exc.py @@ -13,7 +13,11 @@ class BadObject(ODBError): def __str__(self): return "BadObject: %s" % to_hex_sha(self.args[0]) - + +class AmbiguousObjectName(ODBError): + """Thrown if a possibly shortened name does not uniquely represent a single object + in the database""" + class BadObjectType(ODBError): """The object had an unsupported type""" diff --git a/pack.py b/pack.py index 5b13bcc56..ab7c9b677 100644 --- a/pack.py +++ b/pack.py @@ -302,7 +302,53 @@ def sha_to_index(self, sha): # END handle midpoint # END bisect return None - + + def partial_sha_to_index(self, partial_sha): + """:return: index as in `sha_to_index` or None if the sha was not found in this + index file + :param partial_sha: an at least two bytes of a partial sha + :raise AmbiguousObjectName:""" + if len(partial_sha) < 2: + raise ValueError("Require at least 2 bytes of partial sha") + + first_byte = ord(partial_sha[0]) + get_sha = self.sha + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte-1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + len_partial = len(partial_sha) + # fill the partial to full 20 bytes + filled_sha = partial_sha + '\0'*(20 - len_partial) + + # find lowest + while lo < hi: + mid = (lo + hi) / 2 + c = cmp(filled_sha, get_sha(mid)) + if c < 0: + hi = mid + elif not c: + # perfect match + lo = mid + break + else: + lo = mid + 1 + # END handle midpoint + # END bisect + if lo < self.size: + cur_sha = get_sha(lo) + if cur_sha[:len_partial] == partial_sha: + next_sha = None + if lo+1 < self.size: + next_sha = get_sha(lo+1) + if next_sha and next_sha == cur_sha: + raise AmbiguousObjectName(partial_sha) + return lo + # END if we have a match + # END if we found something + return None + if 'PackIndexFile_sha_to_index' in globals(): # NOTE: Its just about 25% faster, the major bottleneck might be the attr # accesses diff --git a/test/db/test_git.py b/test/db/test_git.py index 779e3f15e..1c0c39c96 100644 --- a/test/db/test_git.py +++ b/test/db/test_git.py @@ -1,7 +1,8 @@ from lib import * +from gitdb.exc import BadObject from gitdb.db import GitDB from gitdb.base import OStream, OInfo -from gitdb.util import hex_to_bin +from gitdb.util import hex_to_bin, bin_to_hex class TestGitDB(TestDBBase): @@ -16,7 +17,15 @@ def test_reading(self): assert isinstance(gdb.info(gitdb_sha), OInfo) assert isinstance(gdb.stream(gitdb_sha), OStream) assert gdb.size() > 200 - assert len(list(gdb.sha_iter())) == gdb.size() + sha_list = list(gdb.sha_iter()) + assert len(sha_list) == gdb.size() + + # test partial shas + for binsha in sha_list: + assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8]) == binsha + # END for each sha + + self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") @with_rw_directory def test_writing(self, path): diff --git a/test/db/test_loose.py b/test/db/test_loose.py index 536b02048..8e8a9bfc3 100644 --- a/test/db/test_loose.py +++ b/test/db/test_loose.py @@ -1,10 +1,12 @@ from lib import * from gitdb.db import LooseObjectDB +from gitdb.exc import BadObject +from gitdb.util import bin_to_hex class TestLooseDB(TestDBBase): @with_rw_directory - def test_writing(self, path): + def test_basics(self, path): ldb = LooseObjectDB(path) # write data @@ -16,3 +18,13 @@ def test_writing(self, path): assert shas and len(shas[0]) == 20 assert len(shas) == ldb.size() + + # verify find short object + long_sha = bin_to_hex(shas[-1]) + for short_sha in (long_sha[:20], long_sha[:5]): + assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha + # END for each sha + + self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') + # raises if no object could be foudn + diff --git a/test/db/test_pack.py b/test/db/test_pack.py index f347f408b..c8974b29b 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -2,6 +2,8 @@ from gitdb.db import PackedDB from gitdb.test.lib import fixture_path +from gitdb.exc import BadObject, AmbiguousObjectName + import os import random @@ -44,3 +46,25 @@ def test_writing(self, path): info = pdb.info(sha) stream = pdb.stream(sha) # END for each sha to query + + + # test short finding - be a bit more brutal here + max_bytes = 19 + min_bytes = 2 + num_ambiguous = 0 + for i, sha in enumerate(sha_list): + short_sha = sha[:max((i % max_bytes), min_bytes)] + try: + assert pdb.partial_to_complete_sha(short_sha) == sha + except AmbiguousObjectName: + num_ambiguous += 1 + pass # valid, we can have short objects + # END exception handling + # END for each sha to find + + # we should have at least one ambiguous, considering the small sizes + # but in our pack, there is no ambigious ... + # assert num_ambiguous + + # non-existing + self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0") diff --git a/test/test_pack.py b/test/test_pack.py index 445ed4a17..a9db19178 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -57,7 +57,13 @@ def _assert_index_file(self, index, version, size): assert entry[0] == index.offset(oidx) assert entry[1] == sha assert entry[2] == index.crc(oidx) + + # verify partial sha + for l in (4,8,11,17,20): + assert index.partial_sha_to_index(sha[:l]) == oidx + # END for each object index in indexfile + self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0") def _assert_pack_file(self, pack, version, size): From e9e0496ea982f7574c6ea379e9f6e85dd111fc8e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jul 2010 11:10:25 +0200 Subject: [PATCH 0051/1392] Moved partial_to_complete_sha_hex from gitdb to compounddb, its fitting there better --- db/base.py | 61 +++++++++++++++++++++++++++++++++++++++++++-- db/git.py | 56 ----------------------------------------- test/db/test_git.py | 11 ++++++-- 3 files changed, 68 insertions(+), 60 deletions(-) diff --git a/db/base.py b/db/base.py index 02584c635..ebd5040a5 100644 --- a/db/base.py +++ b/db/base.py @@ -2,10 +2,14 @@ from gitdb.util import ( pool, join, - LazyMixin + LazyMixin, + hex_to_bin ) -from gitdb.exc import BadObject +from gitdb.exc import ( + BadObject, + AmbiguousObjectName + ) from async import ( ChannelThreadTask @@ -186,6 +190,22 @@ def update_cache(self, force=False): # END interface + + +def _databases_recursive(database, output): + """Fill output list with database from db, in order. Deals with Loose, Packed + and compound databases.""" + if isinstance(database, CompoundDB): + compounds = list() + dbs = database.databases() + output.extend(db for db in dbs if not isinstance(db, CompoundDB)) + for cdb in (db for db in dbs if isinstance(db, CompoundDB)): + _databases_recursive(cdb, output) + else: + output.append(database) + # END handle database type + + class CompoundDB(ObjectDBR, LazyMixin, CachingDB): """A database which delegates calls to sub-databases. @@ -258,6 +278,43 @@ def update_cache(self, force=False): # END if is caching db # END for each database to update return stat + + def partial_to_complete_sha_hex(self, partial_hexsha): + """ + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha + :param partial_hexsha: hexsha with less than 40 byte + :raise AmbiguousObjectName: """ + databases = list() + _databases_recursive(self, databases) + + if len(partial_hexsha) % 2 != 0: + partial_binsha = hex_to_bin(partial_hexsha + "0") + else: + partial_binsha = hex_to_bin(partial_hexsha) + # END assure successful binary conversion + + candidate = None + for db in databases: + full_bin_sha = None + try: + if hasattr(db, 'partial_to_complete_sha_hex'): + full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) + else: + full_bin_sha = db.partial_to_complete_sha(partial_binsha) + # END handle database type + except BadObject: + continue + # END ignore bad objects + if full_bin_sha: + if candidate and candidate != full_bin_sha: + raise AmbiguousObjectName(partial_hexsha) + candidate = full_bin_sha + # END handle candidate + # END for each db + if not candidate: + raise BadObject(partial_binsha) + return candidate + #} END interface diff --git a/db/git.py b/db/git.py index 62fed7fb6..f0e63b15a 100644 --- a/db/git.py +++ b/db/git.py @@ -16,26 +16,9 @@ ) import os -from gitdb.util import hex_to_bin - __all__ = ('GitDB', ) -def _databases_recursive(database, output): - """Fill output list with database from db, in order. Deals with Loose, Packed - and compound databases.""" - if isinstance(database, CompoundDB): - compounds = list() - dbs = database.databases() - output.extend(db for db in dbs if not isinstance(db, CompoundDB)) - for cdb in (db for db in dbs if isinstance(db, CompoundDB)): - _databases_recursive(cdb, output) - else: - output.append(database) - # END handle database type - - - class GitDB(FileDBBase, ObjectDBW, CompoundDB): """A git-style object database, which contains all objects in the 'objects' subdirectory""" @@ -96,42 +79,3 @@ def set_ostream(self, ostream): #} END objectdbw interface - #{ Interface - - def partial_to_complete_sha_hex(self, partial_hexsha): - """ - :return: 20 byte binary sha1 from the given less-than-40 byte hexsha - :param partial_hexsha: hexsha with less than 40 byte - :raise AmbiguousObjectName: """ - databases = list() - _databases_recursive(self, databases) - - if len(partial_hexsha) % 2 != 0: - partial_binsha = hex_to_bin(partial_hexsha + "0") - else: - partial_binsha = hex_to_bin(partial_hexsha) - # END assure successful binary conversion - - candidate = None - for db in databases: - full_bin_sha = None - try: - if isinstance(db, LooseObjectDB): - full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) - else: - full_bin_sha = db.partial_to_complete_sha(partial_binsha) - # END handle database type - except BadObject: - continue - # END ignore bad objects - if full_bin_sha: - if candidate and candidate != full_bin_sha: - raise AmbiguousObjectName(partial_hexsha) - candidate = full_bin_sha - # END handle candidate - # END for each db - if not candidate: - raise BadObject(partial_binsha) - return candidate - - #} END interface diff --git a/test/db/test_git.py b/test/db/test_git.py index 1c0c39c96..d2ae10bad 100644 --- a/test/db/test_git.py +++ b/test/db/test_git.py @@ -20,9 +20,16 @@ def test_reading(self): sha_list = list(gdb.sha_iter()) assert len(sha_list) == gdb.size() + + # This is actually a test for compound functionality, but it doesn't + # have a separate test module # test partial shas - for binsha in sha_list: - assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8]) == binsha + # this one as uneven and quite short + assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") + + # mix even/uneven hexshas + for i, binsha in enumerate(sha_list): + assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha # END for each sha self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") From ac7d4757ab4041f5f0f5806934130024b098bb82 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jul 2010 12:10:49 +0200 Subject: [PATCH 0052/1392] Fixed bug caused by incorrect comparison of 'filled-up' binary shas, as it would compare even the filling, instead of taking the canonical length into account and hence ignore the last 4 bytes if the original hex sha had an odd length --- db/base.py | 5 +++-- db/pack.py | 8 ++++++-- fun.py | 21 ++++++++++++++++++++- pack.py | 28 ++++++++++++++++------------ test/db/test_pack.py | 4 ++-- test/test_pack.py | 4 ++-- 6 files changed, 49 insertions(+), 21 deletions(-) diff --git a/db/base.py b/db/base.py index ebd5040a5..1914dbbce 100644 --- a/db/base.py +++ b/db/base.py @@ -287,7 +287,8 @@ def partial_to_complete_sha_hex(self, partial_hexsha): databases = list() _databases_recursive(self, databases) - if len(partial_hexsha) % 2 != 0: + len_partial_hexsha = len(partial_hexsha) + if len_partial_hexsha % 2 != 0: partial_binsha = hex_to_bin(partial_hexsha + "0") else: partial_binsha = hex_to_bin(partial_hexsha) @@ -300,7 +301,7 @@ def partial_to_complete_sha_hex(self, partial_hexsha): if hasattr(db, 'partial_to_complete_sha_hex'): full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) else: - full_bin_sha = db.partial_to_complete_sha(partial_binsha) + full_bin_sha = db.partial_to_complete_sha(partial_binsha, len_partial_hexsha) # END handle database type except BadObject: continue diff --git a/db/pack.py b/db/pack.py index 47d511378..0ec8a4e3b 100644 --- a/db/pack.py +++ b/db/pack.py @@ -177,13 +177,17 @@ def entities(self): """:return: list of pack entities operated upon by this database""" return [ item[1] for item in self._entities ] - def partial_to_complete_sha(self, partial_binsha): + def partial_to_complete_sha(self, partial_binsha, canonical_length): """:return: 20 byte sha as inferred by the given partial binary sha + :param partial_binsha: binary sha with less than 20 bytes + :param canonical_length: length of the corresponding canonical representation. + It is required as binary sha's cannot display whether the original hex sha + had an odd or even number of characters :raise AmbiguousObjectName: :raise BadObject: """ candidate = None for item in self._entities: - item_index = item[1].index().partial_sha_to_index(partial_binsha) + item_index = item[1].index().partial_sha_to_index(partial_binsha, canonical_length) if item_index is not None: sha = item[1].index().sha(item_index) if candidate and candidate != sha: diff --git a/fun.py b/fun.py index ccd8c0fc5..e8bc35531 100644 --- a/fun.py +++ b/fun.py @@ -40,7 +40,8 @@ chunk_size = 1000*mmap.PAGESIZE __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', - 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data' ) + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', + 'is_equal_canonical_sha' ) #{ Routines @@ -223,5 +224,23 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi # yes, lets use the exact same error message that git uses :) assert i == delta_buf_size, "delta replay has gone wild" + +def is_equal_canonical_sha(canonical_length, match, sha1): + """ + :return: True if the given lhs and rhs 20 byte binary shas + The comparison will take the canonical_length of the match sha into account, + hence the comparison will only use the last 4 bytes for uneven canonical representations + :param match: less than 20 byte sha + :param sha1: 20 byte sha""" + binary_length = canonical_length/2 + if match[:binary_length] != sha1[:binary_length]: + return False + + if canonical_length - binary_length and \ + (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: + return False + # END handle uneven canonnical length + return True + #} END routines diff --git a/pack.py b/pack.py index ab7c9b677..6966c295f 100644 --- a/pack.py +++ b/pack.py @@ -12,6 +12,7 @@ from fun import ( pack_object_header_info, + is_equal_canonical_sha, type_id_to_type_map, write_object, stream_copy, @@ -303,24 +304,26 @@ def sha_to_index(self, sha): # END bisect return None - def partial_sha_to_index(self, partial_sha): - """:return: index as in `sha_to_index` or None if the sha was not found in this - index file - :param partial_sha: an at least two bytes of a partial sha + def partial_sha_to_index(self, partial_bin_sha, canonical_length): + """ + :return: index as in `sha_to_index` or None if the sha was not found in this + index file + :param partial_bin_sha: an at least two bytes of a partial binary sha + :param canonical_length: lenght of the original hexadecimal representation of the + given partial binary sha :raise AmbiguousObjectName:""" - if len(partial_sha) < 2: + if len(partial_bin_sha) < 2: raise ValueError("Require at least 2 bytes of partial sha") - first_byte = ord(partial_sha[0]) + first_byte = ord(partial_bin_sha[0]) get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: lo = self._fanout_table[first_byte-1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - len_partial = len(partial_sha) # fill the partial to full 20 bytes - filled_sha = partial_sha + '\0'*(20 - len_partial) + filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) # find lowest while lo < hi: @@ -336,14 +339,15 @@ def partial_sha_to_index(self, partial_sha): lo = mid + 1 # END handle midpoint # END bisect - if lo < self.size: + + if lo < self.size(): cur_sha = get_sha(lo) - if cur_sha[:len_partial] == partial_sha: + if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): next_sha = None - if lo+1 < self.size: + if lo+1 < self.size(): next_sha = get_sha(lo+1) if next_sha and next_sha == cur_sha: - raise AmbiguousObjectName(partial_sha) + raise AmbiguousObjectName(partial_bin_sha) return lo # END if we have a match # END if we found something diff --git a/test/db/test_pack.py b/test/db/test_pack.py index c8974b29b..0386b3f80 100644 --- a/test/db/test_pack.py +++ b/test/db/test_pack.py @@ -55,7 +55,7 @@ def test_writing(self, path): for i, sha in enumerate(sha_list): short_sha = sha[:max((i % max_bytes), min_bytes)] try: - assert pdb.partial_to_complete_sha(short_sha) == sha + assert pdb.partial_to_complete_sha(short_sha, len(short_sha)*2) == sha except AmbiguousObjectName: num_ambiguous += 1 pass # valid, we can have short objects @@ -67,4 +67,4 @@ def test_writing(self, path): # assert num_ambiguous # non-existing - self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0") + self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) diff --git a/test/test_pack.py b/test/test_pack.py index a9db19178..717d07e46 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -60,10 +60,10 @@ def _assert_index_file(self, index, version, size): # verify partial sha for l in (4,8,11,17,20): - assert index.partial_sha_to_index(sha[:l]) == oidx + assert index.partial_sha_to_index(sha[:l], l*2) == oidx # END for each object index in indexfile - self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0") + self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) def _assert_pack_file(self, pack, version, size): From f534e6e9a24f2ac7e7e0f3679551b512d4af569a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 8 Jul 2010 11:24:20 +0200 Subject: [PATCH 0053/1392] Added fixes to setup.py to allow easy_installation --- ext/async | 2 +- setup.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/ext/async b/ext/async index 76f15fc4b..081978422 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 76f15fc4b3e3ccb0160d6c887181f29095d16f23 +Subproject commit 0819784229dc98f92d2c57d740c9aebd533846d6 diff --git a/setup.py b/setup.py index b6c2c414b..1afeb3e17 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,60 @@ #!/usr/bin/env python from distutils.core import setup, Extension - +from distutils.command.build_py import build_py + +import os, sys + +# wow, this is a mixed bag ... I am pretty upset about all of this ... +setuptools_build_py_module = None +try: + # don't pull it in if we don't have to + if 'setuptools' in sys.modules: + import setuptools.command.build_py as setuptools_build_py_module +except ImportError: + pass + +def get_data_files(self): + """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, + the line dealing with the ``plen`` has a bug which causes it to truncate too much. + It is fixed in the system interpreters as they receive patches, and shows how + bad it is if something doesn't have proper unittests. + The code here is a plain copy of the python2.6 version which works for all. + + Generate list of '(package,src_dir,build_dir,filenames)' tuples""" + data = [] + if not self.packages: + return data + + # this one is just for the setup tools ! They don't iniitlialize this variable + # when they should, but do it on demand using this method.Its crazy + if hasattr(self, 'analyze_manifest'): + self.analyze_manifest() + # END handle setuptools ... + + for package in self.packages: + # Locate package source directory + src_dir = self.get_package_dir(package) + + # Compute package build directory + build_dir = os.path.join(*([self.build_lib] + package.split('.'))) + + # Length of path to strip from found files + plen = 0 + if src_dir: + plen = len(src_dir)+1 + + # Strip directory from globbed filenames + filenames = [ + file[plen:] for file in self.find_data_files(package, src_dir) + ] + data.append((package, src_dir, build_dir, filenames)) + return data + +build_py.get_data_files = get_data_files +if setuptools_build_py_module: + setuptools_build_py_module.build_py._get_data_files = get_data_files +# END apply setuptools patch too + setup(name = "gitdb", version = "0.5.0", description = "Git Object Database", @@ -14,5 +68,6 @@ ext_modules=[Extension('gitdb._fun', ['_fun.c'])], license = "BSD License", requires=('async (>=0.6.0)',), + install_requires='async >= 0.6.0', long_description = """GitDB is a pure-Python git object database""" ) From ebf1ead5fb593bc559a96581b60f05165967d35d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 9 Jul 2010 11:14:47 +0200 Subject: [PATCH 0054/1392] stream: adjusted code to allow compilation in python 2.4. It doesn't work though as the mmap implementation isn't advanced enough, but parent projects can at least import gitdb, and possibly use different implementations on demand --- ext/async | 2 +- stream.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/async b/ext/async index 081978422..2842d1eae 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 0819784229dc98f92d2c57d740c9aebd533846d6 +Subproject commit 2842d1eaee8411f7d09c6dc533465b2a404740ec diff --git a/stream.py b/stream.py index 675ccf27e..90bfc996d 100644 --- a/stream.py +++ b/stream.py @@ -167,7 +167,7 @@ def compressed_bytes_read(self): #} END interface - def seek(self, offset, whence=os.SEEK_SET): + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" if offset != 0 or whence != os.SEEK_SET: @@ -407,7 +407,7 @@ def read(self, count=0): self._br += len(data) return data - def seek(self, offset, whence=os.SEEK_SET): + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" @@ -517,7 +517,7 @@ def write(self, data): def close(self): self.buf.write(self.zip.flush()) - def seek(self, offset, whence=os.SEEK_SET): + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Seeking currently only supports to rewind written data Multiple writes are not supported""" if offset != 0 or whence != os.SEEK_SET: From 18152febd428e67b86bb4fb68ec1691d4de75a9c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 9 Jul 2010 11:18:06 +0200 Subject: [PATCH 0055/1392] Bumped version to 0.5.1, added changelog to documentation --- doc/source/changes.rst | 12 ++++++++++++ doc/source/conf.py | 2 +- doc/source/index.rst | 1 + setup.py | 6 +++--- stream.py | 6 +++--- util.py | 43 ++++++++++++++++++++++++++++++++++++++---- 6 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 doc/source/changes.rst diff --git a/doc/source/changes.rst b/doc/source/changes.rst new file mode 100644 index 000000000..84738ae05 --- /dev/null +++ b/doc/source/changes.rst @@ -0,0 +1,12 @@ +######### +Changelog +######### +***** +0.5.1 +***** +* Restored most basic python 2.4 compatibility, such that gitdb can be imported within python 2.4, pack access cannot work though. This at least allows Super-Projects to provide their own workarounds, or use everything but pack support. + +***** +0.5.0 +***** +Initial Release diff --git a/doc/source/conf.py b/doc/source/conf.py index 8e2585c2f..e10addb8d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -47,7 +47,7 @@ # The short X.Y version. version = '0.5' # The full version, including alpha/beta/rc tags. -release = '0.5.0' +release = '0.5.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/index.rst b/doc/source/index.rst index 409c78e99..d414cb22f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -14,6 +14,7 @@ Contents: intro tutorial api + changes Indices and tables ================== diff --git a/setup.py b/setup.py index 1afeb3e17..73d39d09f 100755 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ def get_data_files(self): # END apply setuptools patch too setup(name = "gitdb", - version = "0.5.0", + version = "0.5.1", description = "Git Object Database", author = "Sebastian Thiel", author_email = "byronimo@gmail.com", @@ -67,7 +67,7 @@ def get_data_files(self): package_dir = {'gitdb':''}, ext_modules=[Extension('gitdb._fun', ['_fun.c'])], license = "BSD License", - requires=('async (>=0.6.0)',), - install_requires='async >= 0.6.0', + requires=('async (>=0.6.1)',), + install_requires='async >= 0.6.1', long_description = """GitDB is a pure-Python git object database""" ) diff --git a/stream.py b/stream.py index 90bfc996d..3e55c83a4 100644 --- a/stream.py +++ b/stream.py @@ -170,7 +170,7 @@ def compressed_bytes_read(self): def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" - if offset != 0 or whence != os.SEEK_SET: + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset @@ -411,7 +411,7 @@ def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" - if offset != 0 or whence != os.SEEK_SET: + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset self._br = 0 @@ -520,7 +520,7 @@ def close(self): def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Seeking currently only supports to rewind written data Multiple writes are not supported""" - if offset != 0 or whence != os.SEEK_SET: + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset self.buf.seek(0) diff --git a/util.py b/util.py index 6b2f91073..3316c24f7 100644 --- a/util.py +++ b/util.py @@ -3,7 +3,14 @@ import mmap import sys import errno -import cStringIO + +from cStringIO import StringIO + +# in py 2.4, StringIO is only StringI, without write support. +# Hence we must use the python implementation for this +if sys.version_info[1] < 5: + from StringIO import StringIO +# END handle python 2.4 try: import async.mod.zlib as zlib @@ -73,6 +80,29 @@ def unpack_from(fmt, data, offset=0): #} END Aliases +#{ compatibility stuff ... + +class _RandomAccessStringIO(object): + """Wrapper to provide required functionality in case memory maps cannot or may + not be used. This is only really required in python 2.4""" + __slots__ = '_sio' + + def __init__(self, buf=''): + self._sio = StringIO(buf) + + def __getattr__(self, attr): + return getattr(self._sio, attr) + + def __len__(self): + return len(self.getvalue()) + + def __getitem__(self, i): + return self.getvalue()[i] + + def __getslice__(self, start, end): + return self.getvalue()[start:end] + +#} END compatibility stuff ... #{ Routines @@ -94,7 +124,7 @@ def allocate_memory(size): # this of course may fail if the amount of memory is not available in # one chunk - would only be the case in python 2.4, being more likely on # 32 bit systems. - return cStringIO.StringIO("\0"*size) + return _RandomAccessStringIO("\0"*size) # END handle memory allocation @@ -109,7 +139,12 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): try: if allow_mmap: # supports stream and random access - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + except EnvironmentError: + # python 2.4 issue, 0 wants to be the actual size + return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) + # END handle python 2.4 except OSError: pass # END exception handling @@ -117,7 +152,7 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): # read manully contents = os.read(fd, os.fstat(fd).st_size) if stream: - return cStringIO.StringIO(contents) + return _RandomAccessStringIO(contents) return contents def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): From 449e80bf5f04c9ccc3b1b8926b833227d5fe2d5b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 13 Jul 2010 10:06:27 +0200 Subject: [PATCH 0056/1392] byteswapping in offsets method will only be done if required - previously it was always performed even though the byte order of the system didn't require it --- pack.py | 5 ++++- test/test_example.py | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pack.py b/pack.py index 6966c295f..91323fcce 100644 --- a/pack.py +++ b/pack.py @@ -52,6 +52,8 @@ from itertools import izip import array import os +import sys + __all__ = ('PackIndexFile', 'PackFile', 'PackEntity') @@ -272,7 +274,8 @@ def offsets(self): a.fromstring(buffer(self._data, self._pack_offset, self._pack_64_offset - self._pack_offset)) # networkbyteorder to something array likes more - a.byteswap() + if sys.byteorder == 'little': + a.byteswap() return a else: return tuple(self.offset(index) for index in xrange(self.size())) diff --git a/test/test_example.py b/test/test_example.py index dc82436ec..3fc3fcf5e 100644 --- a/test/test_example.py +++ b/test/test_example.py @@ -22,9 +22,13 @@ def test_base(self): assert ldb.has_object(oinfo.binsha) # END for each sha in database # assure we close all files - del(ostream) - del(oinfo) - + try: + del(ostream) + del(oinfo) + except UnboundLocalError: + pass + # END ignore exception if there are no loose objects + data = "my data" istream = IStream("blob", len(data), StringIO(data)) From 425ecf04aa5038c3d46b01ca20de17c51ef6c4e5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 13 Jul 2010 10:52:11 +0200 Subject: [PATCH 0057/1392] Adjusted setup.py to deal more gracefully with build failures of our optional extensions --- setup.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 73d39d09f..265156df2 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from distutils.core import setup, Extension from distutils.command.build_py import build_py +from distutils.command.build_ext import build_ext import os, sys @@ -10,9 +11,19 @@ # don't pull it in if we don't have to if 'setuptools' in sys.modules: import setuptools.command.build_py as setuptools_build_py_module + from setuptools.command.build_ext import build_ext except ImportError: pass +class build_ext_nofail(build_ext): + """Doesn't fail when build our optional extensions""" + def run(self): + try: + build_ext.run(self) + except Exception: + print "Ignored failure when building extensions, pure python modules will be used instead" + # END ignore errors + def get_data_files(self): """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, the line dealing with the ``plen`` has a bug which causes it to truncate too much. @@ -55,7 +66,8 @@ def get_data_files(self): setuptools_build_py_module.build_py._get_data_files = get_data_files # END apply setuptools patch too -setup(name = "gitdb", +setup(cmdclass={'build_ext':build_ext_nofail}, + name = "gitdb", version = "0.5.1", description = "Git Object Database", author = "Sebastian Thiel", From 274c5c89ba973b0ae7ee56424b160417f33d1d5e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 7 Oct 2010 18:56:53 +0200 Subject: [PATCH 0058/1392] Added frame for actual implementation of the aggregation - for now we just implement a forward-merge of the chunks, then the reverse version once there is some more knowledge on how this works --- ext/async | 2 +- fun.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- stream.py | 26 +++++++++++++++-- util.py | 4 +++ 4 files changed, 109 insertions(+), 6 deletions(-) diff --git a/ext/async b/ext/async index 2842d1eae..5992bb6c8 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 2842d1eaee8411f7d09c6dc533465b2a404740ec +Subproject commit 5992bb6c85973ed81c54c71fef42e2413cd29e88 diff --git a/fun.py b/fun.py index e8bc35531..f18d567ac 100644 --- a/fun.py +++ b/fun.py @@ -41,7 +41,48 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha' ) + 'is_equal_canonical_sha', 'apply_delta_chunks', 'reverse_merge_deltas', + 'merge_deltas') + + +#{ Structures + +class DeltaChunk(object): + """Represents a piece of a delta, it can either add new data, or copy existing + one from a source buffer""" + __slots__ = ( + 'to', # start offset in the target buffer in bytes + 'ts', # size of this chunk in the target buffer in bytes + 'so', # start offset in the source buffer in bytes or None + 'data' # chunk of bytes to be added to the target buffer or None + ) + + def __init__(self, to, ts, so, data): + self.to = to + self.ts = ts + self.so = so + self.data = data + + #{ Interface + + def abssize(self): + return self.to + self.ts + + def apply(self, source, target): + """Apply own data to the target buffer + :param source: buffer providing source bytes for copy operations + :param target: target buffer large enough to contain all the changes to be applied""" + if self.data is not None: + # APPEND DATA + pass + else: + # COPY DATA FROM SOURCE + pass + # END handle chunk mode + + #} END interface + +#} END structures #{ Routines @@ -164,10 +205,46 @@ def stream_copy(read, write, size, chunk_size): return dbw +def reverse_merge_deltas(dcl, dstreams): + """Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + :param dcl: list of DeltaChunk objects, may be empty initially, and will be changed + during the merge process + :param dstreams: iterable of delta stream objects. They must be ordered latest first, + hence the delta to be applied last comes first, then its ancestors + :return: None""" + raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") + +def merge_deltas(dcl, dstreams): + """Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + :param dcl: list of DeltaChunk objects, may be empty initially, and will be changed + during the merge process + :param dstreams: iterable of delta stream objects. They must be ordered latest last, + hence the delta to be applied last comes last, its oldest ancestor first + :return: None""" + for ds in dstreams: + buf = ds.read() + i, src_size = msb_size(buf) + i, target_size = msb_size(buf, i) + + # parse the commands + + # END for each delta stream + +def apply_delta_chunks(src_buf, src_buf_size, dcl, target): + """ + Apply data from a delta chunk list and a source buffer to the target stream + + :param src_buf: random access data from which the delta was created + :param src_buf_size: size of the source buffer in bytes + :param delta_buf_size: size fo the delta buffer in bytes + :param target: ostream with a write method""" + + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): """ - Apply data from a delta buffer using a source buffer to the target file, - which will be written to + Apply data from a delta buffer using a source buffer to the target file :param src_buf: random access data from which the delta was created :param src_buf_size: size of the source buffer in bytes diff --git a/stream.py b/stream.py index 3e55c83a4..74dc2b3c4 100644 --- a/stream.py +++ b/stream.py @@ -7,7 +7,9 @@ from fun import ( msb_size, stream_copy, - apply_delta_data, + apply_delta_data, + apply_delta_chunks, + merge_deltas, delta_types ) @@ -320,9 +322,29 @@ def __init__(self, stream_list): self._br = 0 def _set_cache_(self, attr): + # Aggregate all deltas into one delta in reverse order. Hence we take + # the last delta, and reverse-merge its ancestor delta, until we receive + # the final delta data stream. + dcl = list() + reverse_merge_deltas(dcl, self._dstreams) + + if len(dcl) == 0: + self._size = 0 + self._mm_target = allocate_memory(0) + return + # END handle empty list + + self._size = dcl[-1].abssize() + self._mm_target = allocate_memory(self._size) + + bbuf = allocate_memory(self._bstream.size) + stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) + + apply_delta_chunks(bbuf, self._bstream.size, dcl, self._mm_target) + + def _set_cache_old(self, attr): """If we are here, we apply the actual deltas""" - # prefetch information buffer_info_list = list() max_target_size = 0 for dstream in self._dstreams: diff --git a/util.py b/util.py index 3316c24f7..1ea182025 100644 --- a/util.py +++ b/util.py @@ -117,6 +117,10 @@ def make_sha(source=''): def allocate_memory(size): """:return: a file-protocol accessible memory block of the given size""" + if size == 0: + return _RandomAccessStringIO('') + # END handle empty chunks gracefully + try: return mmap.mmap(-1, size) # read-write by default except EnvironmentError: From afefae6de6fee561842c1a09069f9593b6bd62aa Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 7 Oct 2010 22:46:51 +0200 Subject: [PATCH 0059/1392] Implemented everything around the actual merge-algorithm, which appears to be the heart of the whole thing --- fun.py | 236 +++++++++++++++++++++++++++++++++++++++++++++++------- stream.py | 14 ++-- 2 files changed, 214 insertions(+), 36 deletions(-) diff --git a/fun.py b/fun.py index f18d567ac..c2cbed504 100644 --- a/fun.py +++ b/fun.py @@ -41,12 +41,42 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'apply_delta_chunks', 'reverse_merge_deltas', - 'merge_deltas') + 'is_equal_canonical_sha', 'reverse_merge_deltas', + 'merge_deltas', 'DeltaChunkList') #{ Structures +def _trunc_delta(d, size): + """Truncate the given delta to the given size + :param size: size relative to our target offset, may not be 0, must be smaller or equal + to our size""" + if size == 0: + raise ValueError("size to truncate to must not be 0") + if d.ts == size: + return + if size > d.ts: + raise ValueError("Cannot truncate delta 'larger'") + + d.ts = size + + # NOTE: data is truncated automatically when applying the delta + # MUST NOT DO THIS HERE, see _split_delta + +def _move_delta_offset(d, bytes): + """Move the delta by the given amount of bytes, reducing its size so that its + right bound stays static + :param bytes: amount of bytes to move, must be smaller than delta size""" + if bytes >= d.ts: + raise ValueError("Cannot move offset that much") + + d.to += bytes + d.ts -= bytes + if d.data: + d.data = d.data[bytes:] + # END handle data + + class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" @@ -68,20 +98,120 @@ def __init__(self, to, ts, so, data): def abssize(self): return self.to + self.ts - def apply(self, source, target): + def apply(self, source, write): """Apply own data to the target buffer :param source: buffer providing source bytes for copy operations - :param target: target buffer large enough to contain all the changes to be applied""" - if self.data is not None: - # APPEND DATA - pass - else: + :param write: write method to call with data to write""" + if self.data is None: # COPY DATA FROM SOURCE - pass + write(buffer(source, self.so, self.ts)) + else: + # APPEND DATA + # whats faster: if + 4 function calls or just a write with a slice ? + if self.ts < len(self.data): + write(self.data[:self.ts]) + else: + write(self.data) + # END handle truncation # END handle chunk mode #} END interface +def _closest_index(dcl, absofs): + """:return: index at which the given absofs should be inserted. The index points + to the DeltaChunk with a target buffer absofs that equals or is greater than + absofs + :note: global method for performance only, it belongs to DeltaChunkList""" + # TODO: binary search !! + for i,d in enumerate(dcl): + if absofs >= d.to: + return i + # END for each delta absofs + raise AssertionError("Should never be here") + +def _split_delta(dcl, absofs, di=None): + """Split the delta at di into two deltas, adjusting their sizes, absofss and data + accordingly and adding them to the dcl. + :param absofs: absolute absofs at which to split the delta + :param di: a pre-determined delta-index, or None if it should be retrieved + :note: it will not split if it + :return: the closest index which has been split ( usually di if given) + :note: belongs to DeltaChunkList""" + if di is None: + di = _closest_index(dcl, absofs) + + d = dcl[di] + if d.to == absofs or d.abssize() == absofs: + return di + + _trunc_delta(d, absofs - d.to) + + # insert new one + ds = d.abssize() + relsize = absofs - ds + + self.insert(di+1, DeltaChunk( ds, + relsize, + (d.so and ds) or None, + (d.data and d.data[relsize:]) or None)) + # END adjust next one + return di + +def _merge_delta(dcl, d): + """Merge the given DeltaChunk instance into the dcl""" + index = _closest_index(dcl, d.to) + od = dcl[index] + + if d.data is None: + if od.data: + # OVERWRITE DATA + pass + else: + # MERGE SOURCE AREA + pass + # END overwrite data + else: + if od.data: + # MERGE DATA WITH DATA + pass + else: + # INSERT DATA INTO COPY AREA + pass + # END combine or insert data + # END handle chunk mode + + +class DeltaChunkList(list): + """List with special functionality to deal with DeltaChunks""" + + def init(self, size): + """Intialize this instance with chunks defining to fill up size from a base + buffer of equal size""" + if len(self) != 0: + return + # pretend we have one huge delta chunk, which just copies everything + # from source to destination + maxint32 = 2**32 + for x in range(0, size, maxint32): + self.append(DeltaChunk(x, maxint32, x, None)) + # END create copy chunks + offset = x*maxint32 + remainder = size-offset + if remainder: + self.append(DeltaChunk(offset, remainder, offset, None)) + # END handle all done in loop + + def terminate_at(self, size): + """Chops the list at the given size, splitting and removing DeltaNodes + as required""" + di = _closest_index(self, size) + d = self[di] + rsize = size - d.to + if rsize: + _trunc_delta(d, rsize) + # END truncate last node if possible + del(self[di+(rsize!=0):]) + #} END structures #{ Routines @@ -204,12 +334,10 @@ def stream_copy(read, write, size, chunk_size): # END duplicate data return dbw - def reverse_merge_deltas(dcl, dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: list of DeltaChunk objects, may be empty initially, and will be changed - during the merge process + :param dcl: see merge_deltas :param dstreams: iterable of delta stream objects. They must be ordered latest first, hence the delta to be applied last comes first, then its ancestors :return: None""" @@ -218,31 +346,78 @@ def reverse_merge_deltas(dcl, dstreams): def merge_deltas(dcl, dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: list of DeltaChunk objects, may be empty initially, and will be changed + :param dcl: DeltaChunkList, may be empty initially, and will be changed during the merge process :param dstreams: iterable of delta stream objects. They must be ordered latest last, hence the delta to be applied last comes last, its oldest ancestor first :return: None""" for ds in dstreams: - buf = ds.read() - i, src_size = msb_size(buf) - i, target_size = msb_size(buf, i) + db = ds.read() + delta_buf_size = ds.size + + # read header + i, src_size = msb_size(db) + i, target_size = msb_size(db, i) - # parse the commands + if len(dcl) == 0: + dcl.init(target_size) + # END handle empty list + + # interpret opcodes + tbw = 0 # amount of target bytes written + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_size): + break + + _merge_delta(dcl, DeltaChunk(tbw, cp_size, cp_off, None)) + tbw += cp_size + elif c: + # TODO: Concatenate multiple deltachunks + _merge_delta(dcl, DeltaChunk(tbw, c, None, db[i:i+c])) + i += c + tbw += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + dcl.terminate_at(target_size) # END for each delta stream -def apply_delta_chunks(src_buf, src_buf_size, dcl, target): - """ - Apply data from a delta chunk list and a source buffer to the target stream - - :param src_buf: random access data from which the delta was created - :param src_buf_size: size of the source buffer in bytes - :param delta_buf_size: size fo the delta buffer in bytes - :param target: ostream with a write method""" - -def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_file): +def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ Apply data from a delta buffer using a source buffer to the target file @@ -250,10 +425,9 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi :param src_buf_size: size of the source buffer in bytes :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data - :param target_file: file like object to write the result to + :param write: write method taking a chunk of bytes :note: transcribed to python from the similar routine in patch-delta.c""" i = 0 - twrite = target_file.write db = delta_buf while i < delta_buf_size: c = ord(db[i]) @@ -289,9 +463,9 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, target_fi if (rbound < cp_size or rbound > src_buf_size): break - twrite(buffer(src_buf, cp_off, cp_size)) + write(buffer(src_buf, cp_off, cp_size)) elif c: - twrite(db[i:i+c]) + write(db[i:i+c]) i += c else: raise ValueError("unexpected delta opcode 0") diff --git a/stream.py b/stream.py index 74dc2b3c4..9e507c83b 100644 --- a/stream.py +++ b/stream.py @@ -8,8 +8,8 @@ msb_size, stream_copy, apply_delta_data, - apply_delta_chunks, merge_deltas, + DeltaChunkList, delta_types ) @@ -325,8 +325,8 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = list() - reverse_merge_deltas(dcl, self._dstreams) + dcl = DeltaChunkList() + merge_deltas(dcl, self._dstreams) if len(dcl) == 0: self._size = 0 @@ -338,9 +338,13 @@ def _set_cache_(self, attr): self._mm_target = allocate_memory(self._size) bbuf = allocate_memory(self._bstream.size) - stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) + stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) - apply_delta_chunks(bbuf, self._bstream.size, dcl, self._mm_target) + # APPLY CHUNKS + write = self._mm_target.write + for dc in dcl: + dc.apply(bbuf, write) + # END for each deltachunk to apply def _set_cache_old(self, attr): """If we are here, we apply the actual deltas""" From 4d41a878bf0f9171d9a1a44ef885a58c6a025e0d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 Oct 2010 01:07:38 +0200 Subject: [PATCH 0060/1392] Initial version of the merge algorithm - the current implementation will lead to quite some fragementation, but that can be improved on once it works --- fun.py | 131 +++++++++++++++++++++++++++++++++++++++++------------- stream.py | 4 +- 2 files changed, 103 insertions(+), 32 deletions(-) diff --git a/fun.py b/fun.py index c2cbed504..2b1419cbb 100644 --- a/fun.py +++ b/fun.py @@ -47,7 +47,7 @@ #{ Structures -def _trunc_delta(d, size): +def _set_delta_rbound(d, size): """Truncate the given delta to the given size :param size: size relative to our target offset, may not be 0, must be smaller or equal to our size""" @@ -63,7 +63,7 @@ def _trunc_delta(d, size): # NOTE: data is truncated automatically when applying the delta # MUST NOT DO THIS HERE, see _split_delta -def _move_delta_offset(d, bytes): +def _move_delta_lbound(d, bytes): """Move the delta by the given amount of bytes, reducing its size so that its right bound stays static :param bytes: amount of bytes to move, must be smaller than delta size""" @@ -71,6 +71,7 @@ def _move_delta_offset(d, bytes): raise ValueError("Cannot move offset that much") d.to += bytes + d.so += bytes d.ts -= bytes if d.data: d.data = d.data[bytes:] @@ -95,7 +96,7 @@ def __init__(self, to, ts, so, data): #{ Interface - def abssize(self): + def rbound(self): return self.to + self.ts def apply(self, source, write): @@ -129,39 +130,35 @@ def _closest_index(dcl, absofs): # END for each delta absofs raise AssertionError("Should never be here") -def _split_delta(dcl, absofs, di=None): - """Split the delta at di into two deltas, adjusting their sizes, absofss and data - accordingly and adding them to the dcl. - :param absofs: absolute absofs at which to split the delta - :param di: a pre-determined delta-index, or None if it should be retrieved - :note: it will not split if it - :return: the closest index which has been split ( usually di if given) +def _split_delta(dcl, d, di, relofs, insert_offset=0): + """Split the delta at di into two deltas, adjusting their sizes, offsets and data + accordingly and adding the new part to the dcl + :param relofs: relative offset at which to split the delta + :param d: delta chunk to split + :param di: index of d in dcl + :param insert_offset: offset for the new split id + :return: newly created DeltaChunk :note: belongs to DeltaChunkList""" - if di is None: - di = _closest_index(dcl, absofs) - - d = dcl[di] - if d.to == absofs or d.abssize() == absofs: - return di + if relofs > d.ts: + raise ValueError("Cannot split behinds a chunks rbound") - _trunc_delta(d, absofs - d.to) + osize = d.ts - relofs + _set_delta_rbound(d, relofs) # insert new one - ds = d.abssize() - relsize = absofs - ds + drb = d.rbound() - self.insert(di+1, DeltaChunk( ds, - relsize, - (d.so and ds) or None, - (d.data and d.data[relsize:]) or None)) - # END adjust next one - return di + nd = DeltaChunk( drb, + osize, + (d.so and d.so + osize) or None, + (d.data and d.data[osize:]) or None ) -def _merge_delta(dcl, d): - """Merge the given DeltaChunk instance into the dcl""" - index = _closest_index(dcl, d.to) - od = dcl[index] + self.insert(di+1+insert_offset, nd) + return nd +def _handle_merge(ld, rd): + """Optimize the layout of the lhs delta and the rhs delta + TODO: Once the default implementation is working""" if d.data is None: if od.data: # OVERWRITE DATA @@ -173,6 +170,7 @@ def _merge_delta(dcl, d): else: if od.data: # MERGE DATA WITH DATA + # overwrite the data at the respective spot pass else: # INSERT DATA INTO COPY AREA @@ -180,6 +178,79 @@ def _merge_delta(dcl, d): # END combine or insert data # END handle chunk mode +def _merge_delta(dcl, d): + """Merge the given DeltaChunk instance into the dcl + :param d: the DeltaChunk to merge""" + cdi = _closest_index(dcl, d.to) # current delta index + cd = dcl[cdi] # current delta + + # either we go at his spot, or after + # cdi either moves one up, or stays + dcl.insert(di + (d.to > cd.to), d) + cdi += d.to == cd.to + + while True: + # are we larger than the current block + if d.to < cd.to: + if d.rbound() >= cd.rbound(): + # xxx|xxx|x + # remove the current item completely + dcl.pop(cdi) + cdi -= 1 + elif d.rbound() > cd.to: + # MOVE ITS LBOUND + # xxx|x--| + _move_delta_lbound(cd, d.rbound() - cd.to) + break + else: + # WE DON'T OVERLAP IT + # this can possibly happen + assert False, "Wow, this can really happen" + break + # END rbound overlap handling + # END lbound overlap handling + else: + if d.to >= cd.rbound(): + #|---|...xx + break + # END + + if d.rbound() >= cd.rbound(): + if d.to == cd.to: + #|xxx|x + # REMOVE CD + dcl.pop(cdi) + cdi -= 1 + else: + # TRUNCATE CD + #|-xx| + _set_delta_rbound(cd, d.to - cd.to) + # END handle offset special case + elif d.to == cd.to: + #|x--| + # we shift it by our size + _move_delta_lbound(cd, d.ts) + else: + #|-x-| + # SPLIT CD AND LBOUND MOVE ITS SECOND PART + # insert offset is required to insert it after us + nd = _split_delta(dcl, cd, cdi, 1) + _move_delta_lbound(nd, d.ts) + break + # END handle rbound overlap + # END handle overlap + + cdi += 1 + if cdi < len(dcl): + cd = dcl[cdi] + else: + break + # END check for end of list + # while our chunk is not completely done + + + + class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" @@ -208,7 +279,7 @@ def terminate_at(self, size): d = self[di] rsize = size - d.to if rsize: - _trunc_delta(d, rsize) + _set_delta_rbound(d, rsize) # END truncate last node if possible del(self[di+(rsize!=0):]) diff --git a/stream.py b/stream.py index 9e507c83b..3248ae464 100644 --- a/stream.py +++ b/stream.py @@ -326,7 +326,7 @@ def _set_cache_(self, attr): # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. dcl = DeltaChunkList() - merge_deltas(dcl, self._dstreams) + merge_deltas(dcl, reversed(self._dstreams)) if len(dcl) == 0: self._size = 0 @@ -334,7 +334,7 @@ def _set_cache_(self, attr): return # END handle empty list - self._size = dcl[-1].abssize() + self._size = dcl[-1].rbound() self._mm_target = allocate_memory(self._size) bbuf = allocate_memory(self._bstream.size) From 48fdcf44c536e9ca1008749cf6cd4547303ca519 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 Oct 2010 12:02:15 +0200 Subject: [PATCH 0061/1392] Added new pack to test database to get some ascii deltas, which may possibly help visual debugging ( but probably not ) added plenty of debug code, to realize that the copy operations are not yet correct --- fun.py | 120 +++++++++++------- stream.py | 15 ++- ...bf8e71d8c18879e499335762dd95119d93d9f1.idx | Bin 0 -> 2248 bytes ...f8e71d8c18879e499335762dd95119d93d9f1.pack | Bin 0 -> 3732 bytes test/test_pack.py | 7 +- 5 files changed, 93 insertions(+), 49 deletions(-) create mode 100644 test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx create mode 100644 test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack diff --git a/fun.py b/fun.py index 2b1419cbb..2d51f62eb 100644 --- a/fun.py +++ b/fun.py @@ -10,6 +10,7 @@ decompressobj = zlib.decompressobj import mmap +from itertools import islice, izip # INVARIANTS OFS_DELTA = 6 @@ -93,7 +94,10 @@ def __init__(self, to, ts, so, data): self.ts = ts self.so = so self.data = data - + + def __repr__(self): + return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") + #{ Interface def rbound(self): @@ -105,6 +109,7 @@ def apply(self, source, write): :param write: write method to call with data to write""" if self.data is None: # COPY DATA FROM SOURCE + assert len(source) - self.so - self.ts > 0 write(buffer(source, self.so, self.ts)) else: # APPEND DATA @@ -121,14 +126,16 @@ def apply(self, source, write): def _closest_index(dcl, absofs): """:return: index at which the given absofs should be inserted. The index points to the DeltaChunk with a target buffer absofs that equals or is greater than - absofs + absofs. :note: global method for performance only, it belongs to DeltaChunkList""" # TODO: binary search !! for i,d in enumerate(dcl): - if absofs >= d.to: + if absofs < d.to: + return i-1 + elif absofs == d.to: return i # END for each delta absofs - raise AssertionError("Should never be here") + return len(dcl)-1 def _split_delta(dcl, d, di, relofs, insert_offset=0): """Split the delta at di into two deltas, adjusting their sizes, offsets and data @@ -150,7 +157,7 @@ def _split_delta(dcl, d, di, relofs, insert_offset=0): nd = DeltaChunk( drb, osize, - (d.so and d.so + osize) or None, + d.so + osize, (d.data and d.data[osize:]) or None ) self.insert(di+1+insert_offset, nd) @@ -178,45 +185,51 @@ def _handle_merge(ld, rd): # END combine or insert data # END handle chunk mode -def _merge_delta(dcl, d): +def _merge_delta(dcl, dc): """Merge the given DeltaChunk instance into the dcl :param d: the DeltaChunk to merge""" - cdi = _closest_index(dcl, d.to) # current delta index + if len(dcl) == 0: + dcl.append(dc) + return + # END early return on empty list + + cdi = _closest_index(dcl, dc.to) # current delta index cd = dcl[cdi] # current delta # either we go at his spot, or after # cdi either moves one up, or stays - dcl.insert(di + (d.to > cd.to), d) - cdi += d.to == cd.to + #print "insert at %i" % (cdi + (dc.to > cd.to)) + #print cd, dc + dcl.insert(cdi + (dc.to > cd.to), dc) + cdi += dc.to == cd.to while True: # are we larger than the current block - if d.to < cd.to: - if d.rbound() >= cd.rbound(): + if dc.to < cd.to: + if dc.rbound() >= cd.rbound(): # xxx|xxx|x # remove the current item completely dcl.pop(cdi) cdi -= 1 - elif d.rbound() > cd.to: + elif dc.rbound() > cd.to: # MOVE ITS LBOUND # xxx|x--| - _move_delta_lbound(cd, d.rbound() - cd.to) + _move_delta_lbound(cd, dc.rbound() - cd.to) break else: # WE DON'T OVERLAP IT - # this can possibly happen - assert False, "Wow, this can really happen" + # this can actually happen, once multiple streams are merged break # END rbound overlap handling # END lbound overlap handling else: - if d.to >= cd.rbound(): + if dc.to >= cd.rbound(): #|---|...xx break # END - if d.rbound() >= cd.rbound(): - if d.to == cd.to: + if dc.rbound() >= cd.rbound(): + if dc.to == cd.to: #|xxx|x # REMOVE CD dcl.pop(cdi) @@ -224,18 +237,18 @@ def _merge_delta(dcl, d): else: # TRUNCATE CD #|-xx| - _set_delta_rbound(cd, d.to - cd.to) + _set_delta_rbound(cd, dc.to - cd.to) # END handle offset special case - elif d.to == cd.to: + elif dc.to == cd.to: #|x--| # we shift it by our size - _move_delta_lbound(cd, d.ts) + _move_delta_lbound(cd, dc.ts) else: #|-x-| # SPLIT CD AND LBOUND MOVE ITS SECOND PART # insert offset is required to insert it after us nd = _split_delta(dcl, cd, cdi, 1) - _move_delta_lbound(nd, d.ts) + _move_delta_lbound(nd, dc.ts) break # END handle rbound overlap # END handle overlap @@ -248,30 +261,14 @@ def _merge_delta(dcl, d): # END check for end of list # while our chunk is not completely done - + ## DEBUG ## + dcl.check_integrity() class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" - def init(self, size): - """Intialize this instance with chunks defining to fill up size from a base - buffer of equal size""" - if len(self) != 0: - return - # pretend we have one huge delta chunk, which just copies everything - # from source to destination - maxint32 = 2**32 - for x in range(0, size, maxint32): - self.append(DeltaChunk(x, maxint32, x, None)) - # END create copy chunks - offset = x*maxint32 - remainder = size-offset - if remainder: - self.append(DeltaChunk(offset, remainder, offset, None)) - # END handle all done in loop - def terminate_at(self, size): """Chops the list at the given size, splitting and removing DeltaNodes as required""" @@ -283,6 +280,38 @@ def terminate_at(self, size): # END truncate last node if possible del(self[di+(rsize!=0):]) + ## DEBUG ## + self.check_integrity(size) + + def check_integrity(self, target_size=-1): + """Verify the list has non-overlapping chunks only, and the total size matches + target_size + :param target_size: if not -1, the total size of the chain must be target_size + :raise AssertionError: if the size doen't match""" + if target_size > -1: + assert self[-1].rbound() == target_size + assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size + # END target size verification + + if len(self) < 2: + return + + # check data + for dc in self: + if dc.data: + assert len(dc.data) >= dc.ts + # END for each dc + + left = islice(self, 0, len(self)-1) + right = iter(self) + right.next() + # this is very pythonic - we might have just use index based access here, + # but this could actually be faster + for lft,rgt in izip(left, right): + assert lft.rbound() == rgt.to + assert lft.to + lft.ts == rgt.to + # END for each pair + #} END structures #{ Routines @@ -422,7 +451,8 @@ def merge_deltas(dcl, dstreams): :param dstreams: iterable of delta stream objects. They must be ordered latest last, hence the delta to be applied last comes last, its oldest ancestor first :return: None""" - for ds in dstreams: + for dsi, ds in enumerate(dstreams): + # print "Stream", dsi db = ds.read() delta_buf_size = ds.size @@ -430,10 +460,6 @@ def merge_deltas(dcl, dstreams): i, src_size = msb_size(db) i, target_size = msb_size(db, i) - if len(dcl) == 0: - dcl.init(target_size) - # END handle empty list - # interpret opcodes tbw = 0 # amount of target bytes written while i < delta_buf_size: @@ -475,7 +501,7 @@ def merge_deltas(dcl, dstreams): tbw += cp_size elif c: # TODO: Concatenate multiple deltachunks - _merge_delta(dcl, DeltaChunk(tbw, c, None, db[i:i+c])) + _merge_delta(dcl, DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c else: @@ -487,6 +513,8 @@ def merge_deltas(dcl, dstreams): # END for each delta stream + # print dcl + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ diff --git a/stream.py b/stream.py index 3248ae464..0cb558d78 100644 --- a/stream.py +++ b/stream.py @@ -346,6 +346,19 @@ def _set_cache_(self, attr): dc.apply(bbuf, write) # END for each deltachunk to apply + self._mm_target.seek(0) + + ## DEBUG ## + mt = self._mm_target + for ds in self._dstreams: + ds.stream.seek(0) + self._bstream.stream.seek(0) + self._set_cache_old(attr) + + import chardet + if chardet.detect(mt[:])['encoding'] == 'ascii': + assert self._mm_target[:] == mt[:] + def _set_cache_old(self, attr): """If we are here, we apply the actual deltas""" @@ -399,7 +412,7 @@ def _set_cache_old(self, attr): stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) ####################################################################### - apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf) + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) ####################################################################### # finally, swap out source and target buffers. The target is now the diff --git a/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx b/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx new file mode 100644 index 0000000000000000000000000000000000000000..a7d6c7177ef3ca68b3b93adafa738b190db9e7ac GIT binary patch literal 2248 zcmciEdo&Yz902g;F|X~SRHBG!gxeyIOi6J{XiJoEkwg!9v}mk#)KDpOOOoa!k4*Ad znD;C0xLvwl6=OGfg>ht98CUMtIXTB4?l~RZb9c_??ECxve!uT;e|^834l zlFur*v=R&Nt&}L-{Sjhte~~!ED}jW23nYN#GNd85NCx8XfrfM6>nAw7lq{^5l85zD zt6{z7f2IKM{hM{Lw`e`YixeSV&IZW;C?%+|NEzY<8^75E-~TlgsJTEDY+gbQ;^nAA zegS3~8jxGCWf_`~Tcic?Qd?oYls2rFg^tHg&S9KhKc%fF5m)DGnogOgE9cK~C$WMO z={v>5H?FddyT-|R+8ooZDBBr13G1l(`p)Ymb>YL2_#VbKKEM81Z7lAfb>6yClP)cX z-IsDY8|>}twxKz{$YXc+p$a9R@%Bg9mg5Tp4rL`o3VqnD7g*Ed zx{~AUaaP1Wf~#7IlMkEUflgG5BzXJlX8JYSt7Hf%31j_Amx4J&h32mG<5X|^mz-{v z0B59Ma3tS(ZEeW(Yc-E!!vfb&{P`Nxc%yf*cSCyNO)v6^nzI!_*UE zHlJ`}=uB4W$owt6!*Tv3&qS3bDHnu~FH@nsskcqQ6^Yh)9+XEqU`6y#DxaF#AZomY zb}y}-5-aWau}k?PMsCu_N;ASjne>oFj_GZo;teiRSbcX;+Erd-iiUKOk!4JEa1c4s z!LBr^S8l#V;F>2g;BJ>SniX8xCf#Ug7>;DgZ%!F(eS{i$_wv4a&{e)^jB2xX(bwHY zi5&V?f{?C5NQ?EzRn+3yqhb`_8sHyd8T2N7Ssh||WaC{-qpdi17NhQM^7uf$-~0J$ z2gSUa_3c+BN8gJ5Ws_VKaPso7d>UqT&E$6EnqQ`vt0)Yli(9;s@9m>ET%$f4q^6QY zo@0tLJu1VCqC=a$+#Kq#G(Z39f__z*S3r>t@yYu=X&X=Xbu%qrl(a-w>@VBnQt*d^ z9vMY^FJU+v*+!eJ61g1v)%WKMe;v2nJ#tkAY9j*5n1 zim~2UzDj5TgYDjft#h5;xmG4}2iJygVxpjtQHROSp4b_cQ*Y+>>vb1FM%;j4SF6tu z7FR~L8KoABNT>=c>vCIdshyb|Capo-GR2nm$$kQNJD2W3&>yFDTWH(7c4Tj=HJ!8G zJJCPwUFCJ8WvpYD!HtQYpQm*L(1MB!g5fN>U6+P%?>x&-y#3An^8;APnTL4ijRQuE z$WwdiRFOkHADyPoiJAH^oRaO>Vxig8bk7gI?Cr)DTIXk*lc-7k%4_>eTax^wRjchg z^0wjYKPz$8JQ$9>b?k6-OJ$VmrJG+$)i0FOtwzW}RBHArJC;5D&w6AwZlDG4yHVHRv13FtSg280tXZ zK|8`4da4BUaB&A1dmK1&4(fb>z287M)c6~6(32#Xu>TUkTdV$r_n;@D;dh|nJv1NA x9D{pqvSt~!EoI3lZP9k4v=g3(X<=qg4%Q^Gr|y1tEW5xf@2?fh{cJ?LVl%wKv z^pxJP5=oIZLnI0yh!DSQ+o$<{4#sushM+vou1idydElvH&GMXOwm=VaD??kqO4F_>M_4D*c7}Y%lbdGD7|jlqFbS6b4L_o(2mi^lhrBYBPaBMy20l0zJFs?4ruEBO)n+=jL#_VJqdSJ*20b8ViH{ zwuoTfEIXr<+|m*;6O>E8sC)u#e8JiX>3^v>T18;+nFFhyyy=G{@z)2~*u5*n1C{rz zLiJB?Mn@K=_;jWZ@Y$&d@-4nom^MhLwdiE(Dng#Ba8g{Wf}U#A=gw7cn@eWQl*U;a zA%Ad-&Us{bx93$>Kc})Z8PB_(%a$!|Dng+!lM}12LR|a9D9@u>r*j$j3UwJk#f3rP z?At$`R7-?K2lea!V3~tkm0-X?^!ap>$WT|E$n1$9p!*hYnIAYJUC&PClBwGT!*zMW9Kp(3|Mt9OFsRKKv`Ih0`j{gy98aS<2@FF~Oa>wU zS>YZPeCB{^+gIfnY(H1`z5Ez0<&+f~Njlhu4i?lk&Ql!iJEyNMuo@P7S&d6^wid3o zF*5=eHwqLe0}`P`e4Lk;zi9fHhW48{P8(!5gQTeRZC%gth9B5S*#FD&mu7`_HnFd#N=vNyRVPbT?k?jcZ$ShF6;FjVBt;oo*@jNHr{N&U3Ta4{DqBqd|WWei=TbAWvb_RqhpC5 z$6th{GQK6A2cW+)6g<2_A6-+B#_Qpfd(y_)m``LFyGX|4CQPMOis2qLCh^%`HG?p0 z=(NC%g%K9=u~v+u#+GMk^>?-y<8yw?byu;k^XyPg3J$^kNW`>Meu76zt(Mep%FCiL zXn!Lj?vGbC;C-lpSgV;>SYc_QzQOfkHvH!RxjDfr!jnyX15~VfQnH&F{H{I`GS72L zL{NVjH+I})NuKGYq>Ytel~v%gA_>Zo6|@8+OvrGdIp4aC$X}Sjol)@fBYnWB>&Jtv zh-ga5*&i~oMpORF_b`QYjZoCpzyJesT&kOJGpIPPR_p1n9UkWmrkMsr#bGL~H9UB~ z9hAQQU#cT)W8BKN-Srb(7C3=`Ri3NK%v8>}XlzGR1K|-|CC%{sKQ9AMH^788N~O%2(xw3$ybV#!P%A?&tjj z-|a<&+#jm{eucsYjaWAi$=osckQx3@H?$mgHQ2ceUPPwka`7Jo zjsEu)Di}nAj#wwvYTf#^<6yMG_{0Tq^ADBfe^r*1sPbFwC8nG&9;RmhsjPWSVak#T z5u$HYbzj3^K_mZiEql<6g%6CRZbP!Y7wrSZ>%LcMP~tSN;2QWF-5KcyoCQUxINHlN^&xQTdE!~$W6gR&H9e-KMeP`0wquA@r#MJ$AAYAI8iN*sA?ljAkLtlGW1yE;@t zWQPQ@gfOB6{DOpQ-DM^_JWN_rN9?wkQMY{#1E{LMT>`)(=TsL)R$Vh1jk9`n7fMn{ zdrj)>N?Cnn=1O=~?v!#39CNPT0}2POyzoDP*xiuK*k)K8)tCC$t^Fqbx3e;)y5f@e z`rTpz52?P>R{w2tlRZdy>Ntrzmr4FgsO?z`83cy`!Gh~iyRpR*4usq5cW>dSg&Xk2 zvxy$<7t@wbaWS9HydfP@+nW4%OEr6BgX9&H+TCzarFY4s)o}m1#kb))R)GzEo_6dU z&3j8^drWvetlUi|Kfu!&dQf__hirTQXuLKUqzr@a5Rp-O$Ov_1k(Bb9y>MxVu+ZZ9 z_rleD{DKbpn;Ys~vm3_Kw1EK}FJxFZb5@aa{B6AW!!?tI|86r;UW074U0Pb(>@ zD%-GzhF@C`_uhI%9iyA;o~d)3{5Wv!!LWC_)zFRyFnI$iKw?4@B|2J;#wo#$G{=^V zfdde+6Z#*p@(Dy2$F?Z56^)mj7IRNCZWc<)J)H(#5{TM0Dlb`N#o! z+IxfI3uAJn!^*MdP=h>C*kOXP>0An@7jl_WH81*93j@RFIH)93G%?(+D~O^5(@Anyu2U#}#%!3@3g@`J z{6znRRE5yo1#7NTmHC>oro#hM*G)_-hpM%3Zz9=*lboD}I)XCE)M)H2*AponveGP| zi5d6ORxkyz=*4tWPe8F;T1O}o6{qCkopL_1+Dz-82h8OxALCKtAF?N;^u0ZXcyuxy z!B!uC^wicTVdQd+RWX?(CTy+t6+ZGII^fTZJWZmUo4RIXCac~fge!g9OuKE0GbdIF zydz8E@0h=9S9@rCWP=eX9? zlTwG*y=x;ynHRb=J+FE;pQk^c4`gkN%s->; zSyaOO0IQ^1L`dZ0&Jj@xUmG+_Yn7Sok5_)2>i9b~oxyLOg;w8gTg)x-^md;dSAgNR zxF1cFojbJXB(#C=$+4)azmVi_-}MUDn{uU0DRsA@zk~u$jCO?-UGPgg?=M;|%frgw zLp`aMB7r&MgL0`5WRd3RLFHFVpN>?SiSjL|2nzxEY?RCCHs|M?KOpk)LT9oIS_WNo zTA2NLW4qrsSU-YfXJqs}kwfEl=3duThlD1GT=Zjqc8#!*xP=|O0M2$ln){6^V`&*r zi_;j{HT)+0o1BiHd@?TUJk>1QQ>WMwsD*ET9^PnFR)Z&2-X&IfMgCKfuqx?#!#&8Y z;_MK2OY(%+Xh{&OVJf^ja1KMD&Hkc&pI?b=yGwe%-eT4!9mw=-;K`n>V%Zk}Pv-!JYHwmpvG^M0RM2*)HK7<0FK`<6yhXXOe7pov9#s|RS zL8!y}t=Gt}Lod|sRk;^`p(^~0v3rTdxt;_*3_}uq<9PN5sNo{t7#JBT0GxK9`ojY% zqUv3!pF2LwL9Ir9{+JN2?V;5gu>##-h~L#X$Nd4D1aHL2V*msVS~@J3em+63&olOq z{i=O*Xx#9MLNtr48ADwoyRUCN_W^%m_?s(|d)IS~fp!n;(RCM}$Mnz&*P+9N_J1m1 z^J}M@K0|;G4hX+*{DA>#xq(=I)jb2C1t8=9XF#y=(^;CEbGkV(z+dhn-}3h`voZjv z*#d7IK3tDnYgIZEX|rJY%3(AmV$t{?q;Bw0poR>w@JW*2U*UF=XCV^820)RZ--q?Z zW7qF5)XCtFmMSz;45Us<9w|snM3aVyvXcIMg@=+rv>QWvK@Iru^ zYWBuCP_n@OZR5SKTJG6&DoaSb6jwc$n$k0*sL?jqk>nqCR*diSf}f;>B+#P{K6!XJ zYqsoaUk6 vEUvU}!6G)gfHqQi+dFJwW&vClgS>$17F!H9R$XG(1^|! literal 0 HcmV?d00001 diff --git a/test/test_pack.py b/test/test_pack.py index 717d07e46..770a78bad 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -34,8 +34,10 @@ class TestPack(TestBase): packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) + packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) + packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) def _assert_index_file(self, index, version, size): @@ -123,14 +125,15 @@ def test_pack_index(self): def test_pack(self): # there is this special version 3, but apparently its like 2 ... - for packfile, version, size in (self.packfile_v2_1, self.packfile_v2_2): + for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): pack = PackFile(packfile) self._assert_pack_file(pack, version, size) # END for each pack to test def test_pack_entity(self): for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), - (self.packfile_v2_2, self.packindexfile_v2)): + (self.packfile_v2_2, self.packindexfile_v2), + (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): packfile, version, size = packinfo indexfile, version, size = indexinfo entity = PackEntity(packfile) From 4cf3eac8e9ed6ac2b0bac4a617a08fdf194da4ef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 Oct 2010 19:01:24 +0200 Subject: [PATCH 0062/1392] initial frame for design change - now chunks can refer to DeltaLists as well, which is required to get the copying right. This surely makes things more coplex, but should still result in a performance improvement --- fun.py | 91 ++++++++++++++++++++++++++++++++++++++++++++----------- stream.py | 7 ++--- 2 files changed, 75 insertions(+), 23 deletions(-) diff --git a/fun.py b/fun.py index 2d51f62eb..a0835746e 100644 --- a/fun.py +++ b/fun.py @@ -86,13 +86,14 @@ class DeltaChunk(object): 'to', # start offset in the target buffer in bytes 'ts', # size of this chunk in the target buffer in bytes 'so', # start offset in the source buffer in bytes or None - 'data' # chunk of bytes to be added to the target buffer or None + 'data' # chunk of bytes to be added to the target buffer, + # DeltaChunkList to use as base, or None ) def __init__(self, to, ts, so, data): self.to = to self.ts = ts - self.so = so + self.so = sos self.data = data def __repr__(self): @@ -103,11 +104,15 @@ def __repr__(self): def rbound(self): return self.to + self.ts + def has_data(self): + """:return: True if the instance has data to add to the target stream""" + return self.data is None or not isinstance(self.data, DeltaChunkList) + def apply(self, source, write): """Apply own data to the target buffer :param source: buffer providing source bytes for copy operations :param write: write method to call with data to write""" - if self.data is None: + if self.has_data(): # COPY DATA FROM SOURCE assert len(source) - self.so - self.ts > 0 write(buffer(source, self.so, self.ts)) @@ -166,7 +171,7 @@ def _split_delta(dcl, d, di, relofs, insert_offset=0): def _handle_merge(ld, rd): """Optimize the layout of the lhs delta and the rhs delta TODO: Once the default implementation is working""" - if d.data is None: + if d.has_data(): if od.data: # OVERWRITE DATA pass @@ -217,6 +222,7 @@ def _merge_delta(dcl, dc): _move_delta_lbound(cd, dc.rbound() - cd.to) break else: + # xx.|---| # WE DON'T OVERLAP IT # this can actually happen, once multiple streams are merged break @@ -224,7 +230,7 @@ def _merge_delta(dcl, dc): # END lbound overlap handling else: if dc.to >= cd.rbound(): - #|---|...xx + #|---|xx break # END @@ -269,9 +275,30 @@ def _merge_delta(dcl, dc): class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" - def terminate_at(self, size): + def init(self, size): + """Intialize this instance with chunks defining to fill up size from a base + buffer of equal size + :return: self""" + if len(self) != 0: + return + # pretend we have one huge delta chunk, which just copies everything + # from source to destination + maxint32 = 2**32 + for x in range(0, size, maxint32): + self.append(DeltaChunk(x, maxint32, x, None)) + # END create copy chunks + offset = x*maxint32 + remainder = size-offset + if remainder: + self.append(DeltaChunk(offset, remainder, offset, None)) + # END handle all done in loop + + return self + + def set_rbound(self, size): """Chops the list at the given size, splitting and removing DeltaNodes - as required""" + as required + :return: self""" di = _closest_index(self, size) d = self[di] rsize = size - d.to @@ -283,6 +310,26 @@ def terminate_at(self, size): ## DEBUG ## self.check_integrity(size) + return self + + def connect_with(self, bdlc): + """Connect this instance's delta chunks virtually with the given base. + This means that all copy deltas will simply apply to the given region + of the given base. Afterwards, the base is optimized so that add-deltas + will be truncated to the region actually used, or removed completely where + adequate. This way, memory usage is reduced. + :param bdlc: DeltaChunkList to serve as base""" + raise NotImplementedError("todo") + + def apply(self, bbuf, write): + """Apply the chain's changes and write the final result using the passed + write function. + :param bbuf: base buffer containing the base of all deltas contained in this + list. It will only be used if the chunk in question does not have a base + chain. + :param write: function taking a string of bytes to write to the output""" + raise NotImplementedError("todo") + def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches target_size @@ -437,31 +484,31 @@ def stream_copy(read, write, size, chunk_size): def reverse_merge_deltas(dcl, dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: see merge_deltas + :param dcl: see 3 :param dstreams: iterable of delta stream objects. They must be ordered latest first, hence the delta to be applied last comes first, then its ancestors :return: None""" raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") -def merge_deltas(dcl, dstreams): +def merge_deltas(dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: DeltaChunkList, may be empty initially, and will be changed - during the merge process :param dstreams: iterable of delta stream objects. They must be ordered latest last, hence the delta to be applied last comes last, its oldest ancestor first - :return: None""" + :return: DeltaChunkList, containing all operations to apply""" + bdcl = None # data chunk list for initial base + dcl = DeltaChunkList() for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() delta_buf_size = ds.size # read header - i, src_size = msb_size(db) + i, base_size = msb_size(db) i, target_size = msb_size(db, i) # interpret opcodes - tbw = 0 # amount of target bytes written + tbw = 0 # amount of target bytes written while i < delta_buf_size: c = ord(db[i]) i += 1 @@ -494,14 +541,16 @@ def merge_deltas(dcl, dstreams): rbound = cp_off + cp_size if (rbound < cp_size or - rbound > src_size): + rbound > base_size): break - _merge_delta(dcl, DeltaChunk(tbw, cp_size, cp_off, None)) + # _merge_delta(dcl, DeltaChunk(tbw, cp_size, cp_off, None)) + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: # TODO: Concatenate multiple deltachunks - _merge_delta(dcl, DeltaChunk(tbw, c, 0, db[i:i+c])) + # _merge_delta(dcl, DeltaChunk(tbw, c, 0, db[i:i+c])) + dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c else: @@ -509,8 +558,14 @@ def merge_deltas(dcl, dstreams): # END handle command byte # END while processing delta data - dcl.terminate_at(target_size) + # merge the lists ! + if base is not None: + dcl.connect_with(base) + # END handle merge + # prepare next base + base = dcl + dcl = DeltaChunkList() # END for each delta stream # print dcl diff --git a/stream.py b/stream.py index 0cb558d78..efb99d218 100644 --- a/stream.py +++ b/stream.py @@ -325,8 +325,7 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = DeltaChunkList() - merge_deltas(dcl, reversed(self._dstreams)) + dcl = merge_deltas(reversed(self._dstreams)) if len(dcl) == 0: self._size = 0 @@ -342,9 +341,7 @@ def _set_cache_(self, attr): # APPLY CHUNKS write = self._mm_target.write - for dc in dcl: - dc.apply(bbuf, write) - # END for each deltachunk to apply + dcl.apply(bbuf, write) self._mm_target.seek(0) From fbf8f3221ea58de54567e034d8a7b4e23833e14b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 12:35:35 +0200 Subject: [PATCH 0063/1392] Filled in first implementation of all missing methods. Its untested, and currently the lists are truncated physically, which would work, but it would be easier to just remember the changed bounds, and apply it later with these bounds in mind --- fun.py | 190 ++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 168 insertions(+), 22 deletions(-) diff --git a/fun.py b/fun.py index a0835746e..3e126c3e4 100644 --- a/fun.py +++ b/fun.py @@ -12,6 +12,8 @@ import mmap from itertools import islice, izip +from copy import copy + # INVARIANTS OFS_DELTA = 6 REF_DELTA = 7 @@ -51,33 +53,48 @@ def _set_delta_rbound(d, size): """Truncate the given delta to the given size :param size: size relative to our target offset, may not be 0, must be smaller or equal - to our size""" + to our size + :return: d""" if size == 0: raise ValueError("size to truncate to must not be 0") if d.ts == size: return if size > d.ts: - raise ValueError("Cannot truncate delta 'larger'") + raise ValueError("Cannot extend rbound") d.ts = size # NOTE: data is truncated automatically when applying the delta # MUST NOT DO THIS HERE, see _split_delta + + if d.has_copy_chunklist(): + d.data.set_rbound(size) + # END truncate chunklist + + return d def _move_delta_lbound(d, bytes): """Move the delta by the given amount of bytes, reducing its size so that its right bound stays static - :param bytes: amount of bytes to move, must be smaller than delta size""" + :param bytes: amount of bytes to move, must be smaller than delta size + :return: d""" + if bytes == 0: + return if bytes >= d.ts: raise ValueError("Cannot move offset that much") d.to += bytes d.so += bytes d.ts -= bytes - if d.data: - d.data = d.data[bytes:] + if d.data is not None: + if isinstance(d.data, DeltaChunkList): + d.data.move_lbound(bytes) + else: + d.data = d.data[bytes:] + # END handle data type # END handle data + return d class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing @@ -106,16 +123,22 @@ def rbound(self): def has_data(self): """:return: True if the instance has data to add to the target stream""" - return self.data is None or not isinstance(self.data, DeltaChunkList) + return self.data is not None and not isinstance(self.data, DeltaChunkList) - def apply(self, source, write): + def has_copy_chunklist(self): + """:return: True if we copy our data from a chunklist""" + return return self.data is not None and isinstance(self.data, DeltaChunkList) + + def apply(self, bbuf, write): """Apply own data to the target buffer - :param source: buffer providing source bytes for copy operations + :param bbuf: buffer providing source bytes for copy operations :param write: write method to call with data to write""" - if self.has_data(): + if self.data is None: # COPY DATA FROM SOURCE - assert len(source) - self.so - self.ts > 0 - write(buffer(source, self.so, self.ts)) + assert len(bbuf) - self.so - self.ts > 0 + write(buffer(bbuf, self.so, self.ts)) + elif isinstance(self.data, DeltaChunkList): + self.data.apply(bbuf, write) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? @@ -153,6 +176,8 @@ def _split_delta(dcl, d, di, relofs, insert_offset=0): :note: belongs to DeltaChunkList""" if relofs > d.ts: raise ValueError("Cannot split behinds a chunks rbound") + if relofs < 1: + raise ValueError("Cannot split delta with %i" % relofs) osize = d.ts - relofs _set_delta_rbound(d, relofs) @@ -295,23 +320,65 @@ def init(self, size): return self - def set_rbound(self, size): - """Chops the list at the given size, splitting and removing DeltaNodes + def set_rbound(self, relofs): + """Chops the list at the given relative offset, splitting and removing DeltaNodes as required + :param relofs: offset relative to the start of the chain :return: self""" - di = _closest_index(self, size) + if len(self) == 0: + raise AssertionError("Cannot change bound of empty list") + if relofs == 0: + raise ValueError("Size to truncate to must not be 0") + absofs = self.lbound() + relofs + if absofs > self.rbound(): + raise ValueError("Cannot extend chunk list") + di = _closest_index(self, absofs) d = self[di] - rsize = size - d.to + rsize = absofs - d.to if rsize: _set_delta_rbound(d, rsize) # END truncate last node if possible del(self[di+(rsize!=0):]) ## DEBUG ## - self.check_integrity(size) + self.check_integrity(absofs) return self + def move_lbound(self, bytes): + """Offset the left bound of the list by the given amount of bytes. + This effectively truncates the list + :return: self""" + if len(self) == 0: + raise AssertionError("Cannot change bound of empty list") + if bytes == 0: + return + abslbound = self.lbound() + bytes + if abslbound >= self.rbound(): + raise ValueError("Cannot move lbound that much") + + dsi = _closest_index(self, abslbound) + d = self[dsi] + _move_delta_lbound(d, abslbound - d.to) + + if dsi: + del(self[:dsi]) + # END remove all skipped nodes + + return self + + def rbound(self): + """:return: rightmost extend in bytes, absolute""" + if len(self) == 0: + return 0 + return self[-1].rbound() + + def lbound(self): + """:return: leftmost byte at which this chunklist starts""" + if len(self) == 0: + return 0 + return self[0].to + def connect_with(self, bdlc): """Connect this instance's delta chunks virtually with the given base. This means that all copy deltas will simply apply to the given region @@ -319,7 +386,11 @@ def connect_with(self, bdlc): will be truncated to the region actually used, or removed completely where adequate. This way, memory usage is reduced. :param bdlc: DeltaChunkList to serve as base""" - raise NotImplementedError("todo") + for dc in self: + if not dc.has_data(): + dc.data = bdcl[dc.to, dc.ts] + # END handle overlap + # END for each dc def apply(self, bbuf, write): """Apply the chain's changes and write the final result using the passed @@ -328,7 +399,10 @@ def apply(self, bbuf, write): list. It will only be used if the chunk in question does not have a base chain. :param write: function taking a string of bytes to write to the output""" - raise NotImplementedError("todo") + dapply = DeltaChunk.apply + for dc in self: + dapply(dc, bbuf, write) + # END for each dc def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches @@ -345,6 +419,7 @@ def check_integrity(self, target_size=-1): # check data for dc in self: + assert dc.ts > 0 if dc.data: assert len(dc.data) >= dc.ts # END for each dc @@ -359,6 +434,77 @@ def check_integrity(self, target_size=-1): assert lft.to + lft.ts == rgt.to # END for each pair + def __getslice__(self, absofs, size): + """:return: Subsection of this list at the given absolute offset, with the given + size in bytes. + :return: DeltaChunkList (copy) which represents the given chunk""" + cdi = _closest_index(self, absofs) # delta start index + slen = len(self) + ndcl = self.__class__() + rbound = absofs + size + + while cdi < slen: + # are we larger than the current block + cd = self[cdi] + if absofs < cd.to: + if rbound >= cd.rbound(): + # xxx|xxx|x + # cd is fully contained in the range + ndcl.append(copy(cd)) + elif rbound > cd.to: + # partially contained + # xxx|x--| + cd = copy(cd) + _set_delta_rbound(cd, cd.rbound() - rbound) + ndcl.append(cd) + break + else: + # xx.|---| + # WE DON'T OVERLAP IT + break + # END rbound overlap handling + # END lbound overlap handling + else: + if absofs >= cd.rbound(): + # happens if slice is out of bound + #|---|xx + break + # END + + if rbound >= cd.rbound(): + if absofs == cd.to: + #|xxx|x + # fully contained + ndcl.append(copy(cd)) + else: + # shift + #|-xx| + cd = copy(cd) + _move_delta_lbound(cd, absofs - cd.to) + ndcl.append(cd) + # END handle offset special case + elif absofs == cd.to: + #|x--| + # we truncate it to our size + cd = copy(cd) + _set_delta_rbound(cd, size) + ndcl.append(cd) + break + else: + #|-x-| + # adjust both ends + cd = copy(cd) + _move_delta_lbound(cd, absofs - cd.to) + _set_delta_rbound(cd, size) + ndcl.append(cd) + break + # END handle rbound overlap + # END handle overlap + # END for each chunk + return ndcl + + + #} END structures #{ Routines @@ -559,16 +705,16 @@ def merge_deltas(dstreams): # END while processing delta data # merge the lists ! - if base is not None: - dcl.connect_with(base) + if bdcl is not None: + dcl.connect_with(bdcl) # END handle merge # prepare next base - base = dcl + bdcl = dcl dcl = DeltaChunkList() # END for each delta stream - # print dcl + return base def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): From 1c2caf590d85866d95c3d1470eba692a61de3622 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 19:37:22 +0200 Subject: [PATCH 0064/1392] Forward Delta Application now appears to work --- fun.py | 419 +++++++++++++++++------------------------------------- stream.py | 18 ++- 2 files changed, 146 insertions(+), 291 deletions(-) diff --git a/fun.py b/fun.py index 3e126c3e4..ac0b1c098 100644 --- a/fun.py +++ b/fun.py @@ -44,8 +44,8 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'reverse_merge_deltas', - 'merge_deltas', 'DeltaChunkList') + 'is_equal_canonical_sha', 'reverse_connect_deltas', + 'connect_deltas', 'DeltaChunkList') #{ Structures @@ -55,21 +55,17 @@ def _set_delta_rbound(d, size): :param size: size relative to our target offset, may not be 0, must be smaller or equal to our size :return: d""" - if size == 0: - raise ValueError("size to truncate to must not be 0") if d.ts == size: return + if size == 0: + raise ValueError("size to truncate to must not be 0") if size > d.ts: raise ValueError("Cannot extend rbound") d.ts = size # NOTE: data is truncated automatically when applying the delta - # MUST NOT DO THIS HERE, see _split_delta - - if d.has_copy_chunklist(): - d.data.set_rbound(size) - # END truncate chunklist + # MUST NOT DO THIS HERE return d @@ -85,13 +81,10 @@ def _move_delta_lbound(d, bytes): d.to += bytes d.so += bytes + d.sob += bytes d.ts -= bytes - if d.data is not None: - if isinstance(d.data, DeltaChunkList): - d.data.move_lbound(bytes) - else: - d.data = d.data[bytes:] - # END handle data type + if d.has_data(): + d.data = d.data[bytes:] # END handle data return d @@ -103,14 +96,16 @@ class DeltaChunk(object): 'to', # start offset in the target buffer in bytes 'ts', # size of this chunk in the target buffer in bytes 'so', # start offset in the source buffer in bytes or None - 'data' # chunk of bytes to be added to the target buffer, + 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None + 'sob' # DEBUG: Backup ) def __init__(self, to, ts, so, data): self.to = to self.ts = ts - self.so = sos + self.so = so + self.sob = so self.data = data def __repr__(self): @@ -118,6 +113,18 @@ def __repr__(self): #{ Interface + def copy_offset(self): + """:return: offset to apply when copying from a base buffer, or 0 + if this is not a copying delta chunk""" + + if self.data is not None: + if isinstance(self.data, DeltaChunkList): + return self.data.lbound() + self.so + else: + return self.so + # END handle data type + return 0 + def rbound(self): return self.to + self.ts @@ -127,7 +134,15 @@ def has_data(self): def has_copy_chunklist(self): """:return: True if we copy our data from a chunklist""" - return return self.data is not None and isinstance(self.data, DeltaChunkList) + return self.data is not None and isinstance(self.data, DeltaChunkList) + + def set_copy_chunklist(self, dcl): + """Set the deltachunk list to be used as basis for copying. + :note: only works if this chunk is a copy delta chunk""" + assert self.data is None, "Cannot assign chain to add delta chunk" + self.data = dcl + self.sob = self.so + self.so = 0 # allows lbound moves to be virtual def apply(self, bbuf, write): """Apply own data to the target buffer @@ -135,10 +150,10 @@ def apply(self, bbuf, write): :param write: write method to call with data to write""" if self.data is None: # COPY DATA FROM SOURCE - assert len(bbuf) - self.so - self.ts > 0 + assert len(bbuf) - self.so - self.ts > -1 write(buffer(bbuf, self.so, self.ts)) elif isinstance(self.data, DeltaChunkList): - self.data.apply(bbuf, write) + self.data.apply(bbuf, write, self.so, self.ts) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? @@ -165,208 +180,10 @@ def _closest_index(dcl, absofs): # END for each delta absofs return len(dcl)-1 -def _split_delta(dcl, d, di, relofs, insert_offset=0): - """Split the delta at di into two deltas, adjusting their sizes, offsets and data - accordingly and adding the new part to the dcl - :param relofs: relative offset at which to split the delta - :param d: delta chunk to split - :param di: index of d in dcl - :param insert_offset: offset for the new split id - :return: newly created DeltaChunk - :note: belongs to DeltaChunkList""" - if relofs > d.ts: - raise ValueError("Cannot split behinds a chunks rbound") - if relofs < 1: - raise ValueError("Cannot split delta with %i" % relofs) - - osize = d.ts - relofs - _set_delta_rbound(d, relofs) - - # insert new one - drb = d.rbound() - - nd = DeltaChunk( drb, - osize, - d.so + osize, - (d.data and d.data[osize:]) or None ) - - self.insert(di+1+insert_offset, nd) - return nd - -def _handle_merge(ld, rd): - """Optimize the layout of the lhs delta and the rhs delta - TODO: Once the default implementation is working""" - if d.has_data(): - if od.data: - # OVERWRITE DATA - pass - else: - # MERGE SOURCE AREA - pass - # END overwrite data - else: - if od.data: - # MERGE DATA WITH DATA - # overwrite the data at the respective spot - pass - else: - # INSERT DATA INTO COPY AREA - pass - # END combine or insert data - # END handle chunk mode - -def _merge_delta(dcl, dc): - """Merge the given DeltaChunk instance into the dcl - :param d: the DeltaChunk to merge""" - if len(dcl) == 0: - dcl.append(dc) - return - # END early return on empty list - - cdi = _closest_index(dcl, dc.to) # current delta index - cd = dcl[cdi] # current delta - - # either we go at his spot, or after - # cdi either moves one up, or stays - #print "insert at %i" % (cdi + (dc.to > cd.to)) - #print cd, dc - dcl.insert(cdi + (dc.to > cd.to), dc) - cdi += dc.to == cd.to - - while True: - # are we larger than the current block - if dc.to < cd.to: - if dc.rbound() >= cd.rbound(): - # xxx|xxx|x - # remove the current item completely - dcl.pop(cdi) - cdi -= 1 - elif dc.rbound() > cd.to: - # MOVE ITS LBOUND - # xxx|x--| - _move_delta_lbound(cd, dc.rbound() - cd.to) - break - else: - # xx.|---| - # WE DON'T OVERLAP IT - # this can actually happen, once multiple streams are merged - break - # END rbound overlap handling - # END lbound overlap handling - else: - if dc.to >= cd.rbound(): - #|---|xx - break - # END - - if dc.rbound() >= cd.rbound(): - if dc.to == cd.to: - #|xxx|x - # REMOVE CD - dcl.pop(cdi) - cdi -= 1 - else: - # TRUNCATE CD - #|-xx| - _set_delta_rbound(cd, dc.to - cd.to) - # END handle offset special case - elif dc.to == cd.to: - #|x--| - # we shift it by our size - _move_delta_lbound(cd, dc.ts) - else: - #|-x-| - # SPLIT CD AND LBOUND MOVE ITS SECOND PART - # insert offset is required to insert it after us - nd = _split_delta(dcl, cd, cdi, 1) - _move_delta_lbound(nd, dc.ts) - break - # END handle rbound overlap - # END handle overlap - - cdi += 1 - if cdi < len(dcl): - cd = dcl[cdi] - else: - break - # END check for end of list - # while our chunk is not completely done - - ## DEBUG ## - dcl.check_integrity() - - class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" - def init(self, size): - """Intialize this instance with chunks defining to fill up size from a base - buffer of equal size - :return: self""" - if len(self) != 0: - return - # pretend we have one huge delta chunk, which just copies everything - # from source to destination - maxint32 = 2**32 - for x in range(0, size, maxint32): - self.append(DeltaChunk(x, maxint32, x, None)) - # END create copy chunks - offset = x*maxint32 - remainder = size-offset - if remainder: - self.append(DeltaChunk(offset, remainder, offset, None)) - # END handle all done in loop - - return self - - def set_rbound(self, relofs): - """Chops the list at the given relative offset, splitting and removing DeltaNodes - as required - :param relofs: offset relative to the start of the chain - :return: self""" - if len(self) == 0: - raise AssertionError("Cannot change bound of empty list") - if relofs == 0: - raise ValueError("Size to truncate to must not be 0") - absofs = self.lbound() + relofs - if absofs > self.rbound(): - raise ValueError("Cannot extend chunk list") - di = _closest_index(self, absofs) - d = self[di] - rsize = absofs - d.to - if rsize: - _set_delta_rbound(d, rsize) - # END truncate last node if possible - del(self[di+(rsize!=0):]) - - ## DEBUG ## - self.check_integrity(absofs) - - return self - - def move_lbound(self, bytes): - """Offset the left bound of the list by the given amount of bytes. - This effectively truncates the list - :return: self""" - if len(self) == 0: - raise AssertionError("Cannot change bound of empty list") - if bytes == 0: - return - abslbound = self.lbound() + bytes - if abslbound >= self.rbound(): - raise ValueError("Cannot move lbound that much") - - dsi = _closest_index(self, abslbound) - d = self[dsi] - _move_delta_lbound(d, abslbound - d.to) - - if dsi: - del(self[:dsi]) - # END remove all skipped nodes - - return self - def rbound(self): """:return: rightmost extend in bytes, absolute""" if len(self) == 0: @@ -379,30 +196,84 @@ def lbound(self): return 0 return self[0].to - def connect_with(self, bdlc): + def size(self): + """:return: size of bytes as measured by our delta chunks""" + return self.rbound() - self.lbound() + + def connect_with(self, bdcl): """Connect this instance's delta chunks virtually with the given base. This means that all copy deltas will simply apply to the given region of the given base. Afterwards, the base is optimized so that add-deltas will be truncated to the region actually used, or removed completely where adequate. This way, memory usage is reduced. - :param bdlc: DeltaChunkList to serve as base""" + :param bdcl: DeltaChunkList to serve as base""" for dc in self: if not dc.has_data(): - dc.data = bdcl[dc.to, dc.ts] + # dc.set_copy_chunklist(bdcl[dc.copy_offset():dc.ts]) + dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) # END handle overlap # END for each dc - def apply(self, bbuf, write): + def apply(self, bbuf, write, lbound_offset=0, size=0): """Apply the chain's changes and write the final result using the passed write function. :param bbuf: base buffer containing the base of all deltas contained in this list. It will only be used if the chunk in question does not have a base chain. + :param lbound_offset: offset at which to start applying the delta, relative to + our lbound + :param size: if larger than 0, only the given amount of bytes will be applied :param write: function taking a string of bytes to write to the output""" + slen = len(self) + if slen == 0: + return + # END early abort + absofs = self.lbound() + lbound_offset + if size == 0: + size = self.rbound() - absofs + # END initialize size + if absofs + size > self.rbound(): + raise ValueError("Cannot apply more bytes than there are in this chain") + # END sanity check + + if size > self.rbound() - absofs: + raise ValueError("Trying to apply more than there is available") + dapply = DeltaChunk.apply - for dc in self: - dapply(dc, bbuf, write) - # END for each dc + if lbound_offset or absofs + size != self.rbound(): + cdi = _closest_index(self, absofs) + cd = self[cdi] + if cd.to != absofs: + tcd = copy(cd) + _move_delta_lbound(tcd, absofs - cd.to) + _set_delta_rbound(tcd, min(tcd.ts, size)) + dapply(tcd, bbuf, write) + size -= tcd.ts + cdi += 1 + # END handle first chunk + + # here we have to either apply full chunks, or smaller ones, but + # we always start at the chunks target offset + while cdi < slen and size: + cd = self[cdi] + if cd.ts <= size: + dapply(cd, bbuf, write) + size -= cd.ts + else: + tcd = copy(cd) + _set_delta_rbound(tcd, size) + dapply(tcd, bbuf, write) + size -= tcd.ts + break + # END handle bytes to apply + cdi += 1 + # END handle rest + assert size == 0 + else: + for dc in self: + dapply(dc, bbuf, write) + # END for each dc + # END handle application values def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches @@ -420,8 +291,10 @@ def check_integrity(self, target_size=-1): # check data for dc in self: assert dc.ts > 0 - if dc.data: + if dc.has_data(): assert len(dc.data) >= dc.ts + if dc.has_copy_chunklist(): + assert dc.ts <= dc.data.size() # END for each dc left = islice(self, 0, len(self)-1) @@ -438,69 +311,43 @@ def __getslice__(self, absofs, size): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. :return: DeltaChunkList (copy) which represents the given chunk""" + if len(self) == 0: + return DeltaChunkList() + + absofs = max(absofs, self.lbound()) + size = min(self.rbound() - self.lbound(), size) cdi = _closest_index(self, absofs) # delta start index + cd = self[cdi] slen = len(self) ndcl = self.__class__() - rbound = absofs + size - while cdi < slen: + if cd.to != absofs: + tcd = copy(cd) + _move_delta_lbound(tcd, absofs - cd.to) + _set_delta_rbound(tcd, min(tcd.ts, size)) + ndcl.append(tcd) + size -= tcd.ts + cdi += 1 + # END lbound overlap handling + + while cdi < slen and size: # are we larger than the current block cd = self[cdi] - if absofs < cd.to: - if rbound >= cd.rbound(): - # xxx|xxx|x - # cd is fully contained in the range - ndcl.append(copy(cd)) - elif rbound > cd.to: - # partially contained - # xxx|x--| - cd = copy(cd) - _set_delta_rbound(cd, cd.rbound() - rbound) - ndcl.append(cd) - break - else: - # xx.|---| - # WE DON'T OVERLAP IT - break - # END rbound overlap handling - # END lbound overlap handling + if cd.ts <= size: + ndcl.append(copy(cd)) + size -= cd.ts else: - if absofs >= cd.rbound(): - # happens if slice is out of bound - #|---|xx - break - # END - - if rbound >= cd.rbound(): - if absofs == cd.to: - #|xxx|x - # fully contained - ndcl.append(copy(cd)) - else: - # shift - #|-xx| - cd = copy(cd) - _move_delta_lbound(cd, absofs - cd.to) - ndcl.append(cd) - # END handle offset special case - elif absofs == cd.to: - #|x--| - # we truncate it to our size - cd = copy(cd) - _set_delta_rbound(cd, size) - ndcl.append(cd) - break - else: - #|-x-| - # adjust both ends - cd = copy(cd) - _move_delta_lbound(cd, absofs - cd.to) - _set_delta_rbound(cd, size) - ndcl.append(cd) - break - # END handle rbound overlap - # END handle overlap + tcd = copy(cd) + _set_delta_rbound(tcd, size) + ndcl.append(tcd) + size -= tcd.ts + break + # END hadle size + cdi += 1 # END for each chunk + assert size == 0, "size was %i" % size + + ndcl.check_integrity() return ndcl @@ -627,7 +474,7 @@ def stream_copy(read, write, size, chunk_size): # END duplicate data return dbw -def reverse_merge_deltas(dcl, dstreams): +def reverse_connect_deltas(dcl, dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks :param dcl: see 3 @@ -636,7 +483,7 @@ def reverse_merge_deltas(dcl, dstreams): :return: None""" raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") -def merge_deltas(dstreams): +def connect_deltas(dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks :param dstreams: iterable of delta stream objects. They must be ordered latest last, @@ -690,12 +537,10 @@ def merge_deltas(dstreams): rbound > base_size): break - # _merge_delta(dcl, DeltaChunk(tbw, cp_size, cp_off, None)) dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: # TODO: Concatenate multiple deltachunks - # _merge_delta(dcl, DeltaChunk(tbw, c, 0, db[i:i+c])) dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c @@ -709,12 +554,14 @@ def merge_deltas(dstreams): dcl.connect_with(bdcl) # END handle merge + dcl.check_integrity() + # prepare next base bdcl = dcl dcl = DeltaChunkList() # END for each delta stream - return base + return bdcl def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): diff --git a/stream.py b/stream.py index efb99d218..8b8655e86 100644 --- a/stream.py +++ b/stream.py @@ -8,7 +8,7 @@ msb_size, stream_copy, apply_delta_data, - merge_deltas, + connect_deltas, DeltaChunkList, delta_types ) @@ -325,7 +325,7 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = merge_deltas(reversed(self._dstreams)) + dcl = connect_deltas(reversed(self._dstreams)) if len(dcl) == 0: self._size = 0 @@ -333,7 +333,7 @@ def _set_cache_(self, attr): return # END handle empty list - self._size = dcl[-1].rbound() + self._size = dcl.rbound() self._mm_target = allocate_memory(self._size) bbuf = allocate_memory(self._bstream.size) @@ -353,8 +353,16 @@ def _set_cache_(self, attr): self._set_cache_old(attr) import chardet - if chardet.detect(mt[:])['encoding'] == 'ascii': - assert self._mm_target[:] == mt[:] + + print "num dstreams", len(self._dstreams) + #if chardet.detect(mt[:self._size])['encoding'] == 'ascii': + if self._mm_target[:self._size] != mt[:]: + open("working.txt", "w").write(self._mm_target[:self._size]) + open("incorrect.txt", "w").write(mt[:]) + raise AssertionError("Output didn't match") + # END debug + print "success" + def _set_cache_old(self, attr): """If we are here, we apply the actual deltas""" From 834f081232cb51c251e8f2d3931c9e1fd5eff457 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 23:07:28 +0200 Subject: [PATCH 0065/1392] Implemented add-chunk compression, which clearly reduces chain size, but might not really be worth it in python --- fun.py | 70 +++++++++++++++++++++++++++++++++++++++++-------------- stream.py | 29 +++++++---------------- 2 files changed, 61 insertions(+), 38 deletions(-) diff --git a/fun.py b/fun.py index ac0b1c098..170b3db15 100644 --- a/fun.py +++ b/fun.py @@ -13,6 +13,7 @@ from itertools import islice, izip from copy import copy +from cStringIO import StringIO # INVARIANTS OFS_DELTA = 6 @@ -57,10 +58,6 @@ def _set_delta_rbound(d, size): :return: d""" if d.ts == size: return - if size == 0: - raise ValueError("size to truncate to must not be 0") - if size > d.ts: - raise ValueError("Cannot extend rbound") d.ts = size @@ -76,8 +73,6 @@ def _move_delta_lbound(d, bytes): :return: d""" if bytes == 0: return - if bytes >= d.ts: - raise ValueError("Cannot move offset that much") d.to += bytes d.so += bytes @@ -139,7 +134,6 @@ def has_copy_chunklist(self): def set_copy_chunklist(self, dcl): """Set the deltachunk list to be used as basis for copying. :note: only works if this chunk is a copy delta chunk""" - assert self.data is None, "Cannot assign chain to add delta chunk" self.data = dcl self.sob = self.so self.so = 0 # allows lbound moves to be virtual @@ -150,13 +144,13 @@ def apply(self, bbuf, write): :param write: write method to call with data to write""" if self.data is None: # COPY DATA FROM SOURCE - assert len(bbuf) - self.so - self.ts > -1 write(buffer(bbuf, self.so, self.ts)) elif isinstance(self.data, DeltaChunkList): self.data.apply(bbuf, write, self.so, self.ts) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? + # Considering data can be larger than 127 bytes now, it should be worth it if self.ts < len(self.data): write(self.data[:self.ts]) else: @@ -209,11 +203,54 @@ def connect_with(self, bdcl): :param bdcl: DeltaChunkList to serve as base""" for dc in self: if not dc.has_data(): - # dc.set_copy_chunklist(bdcl[dc.copy_offset():dc.ts]) dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) # END handle overlap # END for each dc + def compress(self): + """Alter the list to reduce the amount of nodes. Currently we concatenate + add-chunks + :return: self""" + slen = len(self) + if slen < 2: + return self + i = 0 + slen_orig = slen + + first_data_index = None + while i < slen: + dc = self[i] + i += 1 + if not dc.has_data(): + if first_data_index is not None and i-2-first_data_index > 1: + #if first_data_index is not None: + nd = StringIO() # new data + so = self[first_data_index].to # start offset in target buffer + for x in xrange(first_data_index, i-1): + xdc = self[x] + nd.write(xdc.data[:xdc.ts]) + # END collect data + + del(self[first_data_index:i-1]) + buf = nd.getvalue() + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + + slen = len(self) + i = first_data_index + 1 + + # END concatenate data + first_data_index = None + continue + # END skip non-data chunks + + if first_data_index is None: + first_data_index = i-1 + # END iterate list + + #if slen_orig != len(self): + # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) + return self + def apply(self, bbuf, write, lbound_offset=0, size=0): """Apply the chain's changes and write the final result using the passed write function. @@ -232,12 +269,6 @@ def apply(self, bbuf, write, lbound_offset=0, size=0): if size == 0: size = self.rbound() - absofs # END initialize size - if absofs + size > self.rbound(): - raise ValueError("Cannot apply more bytes than there are in this chain") - # END sanity check - - if size > self.rbound() - absofs: - raise ValueError("Trying to apply more than there is available") dapply = DeltaChunk.apply if lbound_offset or absofs + size != self.rbound(): @@ -347,7 +378,7 @@ def __getslice__(self, absofs, size): # END for each chunk assert size == 0, "size was %i" % size - ndcl.check_integrity() + # ndcl.check_integrity() return ndcl @@ -540,7 +571,8 @@ def connect_deltas(dstreams): dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: - # TODO: Concatenate multiple deltachunks + # NOTE: in C, the data chunks should probably be concatenated here. + # In python, we do it as a post-process dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c @@ -549,12 +581,14 @@ def connect_deltas(dstreams): # END handle command byte # END while processing delta data + dcl.compress() + # merge the lists ! if bdcl is not None: dcl.connect_with(bdcl) # END handle merge - dcl.check_integrity() + # dcl.check_integrity() # prepare next base bdcl = dcl diff --git a/stream.py b/stream.py index 8b8655e86..098d27a2b 100644 --- a/stream.py +++ b/stream.py @@ -322,6 +322,14 @@ def __init__(self, stream_list): self._br = 0 def _set_cache_(self, attr): + # the direct algorithm is fastest and most direct if there is only one + # delta. Also, the extra overhead might not be worth it for items smaller + # than X - definitely the case in python + #print "num streams", len(self._dstreams) + #if len(self._dstreams) == 1 or (len(self._dstreams) * self._dstreams.size) > 25*1000*1000: + if len(self._dstreams) == 1: + return self._set_cache_brute_(attr) + # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. @@ -345,26 +353,7 @@ def _set_cache_(self, attr): self._mm_target.seek(0) - ## DEBUG ## - mt = self._mm_target - for ds in self._dstreams: - ds.stream.seek(0) - self._bstream.stream.seek(0) - self._set_cache_old(attr) - - import chardet - - print "num dstreams", len(self._dstreams) - #if chardet.detect(mt[:self._size])['encoding'] == 'ascii': - if self._mm_target[:self._size] != mt[:]: - open("working.txt", "w").write(self._mm_target[:self._size]) - open("incorrect.txt", "w").write(mt[:]) - raise AssertionError("Output didn't match") - # END debug - print "success" - - - def _set_cache_old(self, attr): + def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() From bda5ef5e94161c304d6151785b34a20bdb306389 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 23:32:01 +0200 Subject: [PATCH 0066/1392] implemented binary tree search to get the closest deltachunk by offset --- fun.py | 18 ++++++++++++------ test/test_pack.py | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/fun.py b/fun.py index 170b3db15..8f38fa467 100644 --- a/fun.py +++ b/fun.py @@ -165,12 +165,18 @@ def _closest_index(dcl, absofs): to the DeltaChunk with a target buffer absofs that equals or is greater than absofs. :note: global method for performance only, it belongs to DeltaChunkList""" - # TODO: binary search !! - for i,d in enumerate(dcl): - if absofs < d.to: - return i-1 - elif absofs == d.to: - return i + lo = 0 + hi = len(dcl) + while lo < hi: + mid = (lo + hi) / 2 + dc = dcl[mid] + if dc.to > absofs: + hi = mid + elif dc.rbound() > absofs or dc.to == absofs: + return mid + else: + lo = mid + 1 + # END handle bound # END for each delta absofs return len(dcl)-1 diff --git a/test/test_pack.py b/test/test_pack.py index 770a78bad..6e598d755 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -130,7 +130,7 @@ def test_pack(self): self._assert_pack_file(pack, version, size) # END for each pack to test - def test_pack_entity(self): + def _test_pack_entity(self): for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): From 5d18685948602de69eb950d0238fcf80f0413b68 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 10 Oct 2010 23:53:11 +0200 Subject: [PATCH 0067/1392] Disabled delta-aggregation as it is reduces the throughput to 540KiB/s compared to 9.4MiB compared to the previous brute-force algorithm. Compression helps, but it would probably be more efficient if done right away, not as post-process. It might help to implement the reversed version of this algorithm, as initially intended, but currently the overhead is the actual application --- fun.py | 4 ---- stream.py | 11 ++++++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/fun.py b/fun.py index 8f38fa467..bc67e0fce 100644 --- a/fun.py +++ b/fun.py @@ -76,7 +76,6 @@ def _move_delta_lbound(d, bytes): d.to += bytes d.so += bytes - d.sob += bytes d.ts -= bytes if d.has_data(): d.data = d.data[bytes:] @@ -93,14 +92,12 @@ class DeltaChunk(object): 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None - 'sob' # DEBUG: Backup ) def __init__(self, to, ts, so, data): self.to = to self.ts = ts self.so = so - self.sob = so self.data = data def __repr__(self): @@ -135,7 +132,6 @@ def set_copy_chunklist(self, dcl): """Set the deltachunk list to be used as basis for copying. :note: only works if this chunk is a copy delta chunk""" self.data = dcl - self.sob = self.so self.so = 0 # allows lbound moves to be virtual def apply(self, bbuf, write): diff --git a/stream.py b/stream.py index 098d27a2b..7347f527c 100644 --- a/stream.py +++ b/stream.py @@ -311,6 +311,10 @@ class DeltaApplyReader(LazyMixin): "_br" # number of bytes read ) + #{ Configuration + k_max_memory_move = 250*1000*1000 + #} END configuration + def __init__(self, stream_list): """Initialize this instance with a list of streams, the first stream being the delta to apply on top of all following deltas, the last stream being the @@ -325,8 +329,9 @@ def _set_cache_(self, attr): # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller # than X - definitely the case in python - #print "num streams", len(self._dstreams) - #if len(self._dstreams) == 1 or (len(self._dstreams) * self._dstreams.size) > 25*1000*1000: + # hence we apply a worst-case scenario here + # TODO: read the final size from the deltastream - have to partly unpack + # if len(self._dstreams) * self._size < self.k_max_memory_move: if len(self._dstreams) == 1: return self._set_cache_brute_(attr) @@ -353,7 +358,7 @@ def _set_cache_(self, attr): self._mm_target.seek(0) - def _set_cache_brute_(self, attr): + def _set_cache_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() From 682f483fa61c77fd6121ae860002094eb517bbd4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 00:07:37 +0200 Subject: [PATCH 0068/1392] First profiling run revealed that the copy function was a serious slowdown. Now its twice as fast compared to the previous version, but still about 8 times slower than the brute force approach --- fun.py | 14 ++++++++------ stream.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/fun.py b/fun.py index bc67e0fce..9f2c9b6be 100644 --- a/fun.py +++ b/fun.py @@ -12,7 +12,6 @@ import mmap from itertools import islice, izip -from copy import copy from cStringIO import StringIO # INVARIANTS @@ -82,6 +81,9 @@ def _move_delta_lbound(d, bytes): # END handle data return d + +def delta_duplicate(src): + return DeltaChunk(src.to, src.ts, src.so, src.data) class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing @@ -277,7 +279,7 @@ def apply(self, bbuf, write, lbound_offset=0, size=0): cdi = _closest_index(self, absofs) cd = self[cdi] if cd.to != absofs: - tcd = copy(cd) + tcd = delta_duplicate(cd) _move_delta_lbound(tcd, absofs - cd.to) _set_delta_rbound(tcd, min(tcd.ts, size)) dapply(tcd, bbuf, write) @@ -293,7 +295,7 @@ def apply(self, bbuf, write, lbound_offset=0, size=0): dapply(cd, bbuf, write) size -= cd.ts else: - tcd = copy(cd) + tcd = delta_duplicate(cd) _set_delta_rbound(tcd, size) dapply(tcd, bbuf, write) size -= tcd.ts @@ -355,7 +357,7 @@ def __getslice__(self, absofs, size): ndcl = self.__class__() if cd.to != absofs: - tcd = copy(cd) + tcd = delta_duplicate(cd) _move_delta_lbound(tcd, absofs - cd.to) _set_delta_rbound(tcd, min(tcd.ts, size)) ndcl.append(tcd) @@ -367,10 +369,10 @@ def __getslice__(self, absofs, size): # are we larger than the current block cd = self[cdi] if cd.ts <= size: - ndcl.append(copy(cd)) + ndcl.append(delta_duplicate(cd)) size -= cd.ts else: - tcd = copy(cd) + tcd = delta_duplicate(cd) _set_delta_rbound(tcd, size) ndcl.append(tcd) size -= tcd.ts diff --git a/stream.py b/stream.py index 7347f527c..16c917a97 100644 --- a/stream.py +++ b/stream.py @@ -358,7 +358,7 @@ def _set_cache_(self, attr): self._mm_target.seek(0) - def _set_cache_(self, attr): + def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() From a064007f0e25b8a4af04c30fe329e345edc8092e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 08:08:09 +0200 Subject: [PATCH 0069/1392] Made heavliy called methods global, its brings a second, which is nearly 10 percent more performance just by eliminating two method calls --- fun.py | 154 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 79 insertions(+), 75 deletions(-) diff --git a/fun.py b/fun.py index 9f2c9b6be..253e64092 100644 --- a/fun.py +++ b/fun.py @@ -84,6 +84,26 @@ def _move_delta_lbound(d, bytes): def delta_duplicate(src): return DeltaChunk(src.to, src.ts, src.so, src.data) + +def delta_chunk_apply(dc, bbuf, write): + """Apply own data to the target buffer + :param bbuf: buffer providing source bytes for copy operations + :param write: write method to call with data to write""" + if dc.data is None: + # COPY DATA FROM SOURCE + write(buffer(bbuf, dc.so, dc.ts)) + elif isinstance(dc.data, DeltaChunkList): + delta_list_apply(dc.data, bbuf, write, dc.so, dc.ts) + else: + # APPEND DATA + # whats faster: if + 4 function calls or just a write with a slice ? + # Considering data can be larger than 127 bytes now, it should be worth it + if dc.ts < len(dc.data): + write(dc.data[:dc.ts]) + else: + write(dc.data) + # END handle truncation + # END handle chunk mode class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing @@ -136,25 +156,7 @@ def set_copy_chunklist(self, dcl): self.data = dcl self.so = 0 # allows lbound moves to be virtual - def apply(self, bbuf, write): - """Apply own data to the target buffer - :param bbuf: buffer providing source bytes for copy operations - :param write: write method to call with data to write""" - if self.data is None: - # COPY DATA FROM SOURCE - write(buffer(bbuf, self.so, self.ts)) - elif isinstance(self.data, DeltaChunkList): - self.data.apply(bbuf, write, self.so, self.ts) - else: - # APPEND DATA - # whats faster: if + 4 function calls or just a write with a slice ? - # Considering data can be larger than 127 bytes now, it should be worth it - if self.ts < len(self.data): - write(self.data[:self.ts]) - else: - write(self.data) - # END handle truncation - # END handle chunk mode + #} END interface @@ -178,6 +180,59 @@ def _closest_index(dcl, absofs): # END for each delta absofs return len(dcl)-1 +def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): + """Apply the chain's changes and write the final result using the passed + write function. + :param bbuf: base buffer containing the base of all deltas contained in this + list. It will only be used if the chunk in question does not have a base + chain. + :param lbound_offset: offset at which to start applying the delta, relative to + our lbound + :param size: if larger than 0, only the given amount of bytes will be applied + :param write: function taking a string of bytes to write to the output""" + slen = len(dcl) + if slen == 0: + return + # END early abort + absofs = dcl.lbound() + lbound_offset + if size == 0: + size = dcl.rbound() - absofs + # END initialize size + + if lbound_offset or absofs + size != dcl.rbound(): + cdi = _closest_index(dcl, absofs) + cd = dcl[cdi] + if cd.to != absofs: + tcd = delta_duplicate(cd) + _move_delta_lbound(tcd, absofs - cd.to) + _set_delta_rbound(tcd, min(tcd.ts, size)) + delta_chunk_apply(tcd, bbuf, write) + size -= tcd.ts + cdi += 1 + # END handle first chunk + + # here we have to either apply full chunks, or smaller ones, but + # we always start at the chunks target offset + while cdi < slen and size: + cd = dcl[cdi] + if cd.ts <= size: + delta_chunk_apply(cd, bbuf, write) + size -= cd.ts + else: + tcd = delta_duplicate(cd) + _set_delta_rbound(tcd, size) + delta_chunk_apply(tcd, bbuf, write) + size -= tcd.ts + break + # END handle bytes to apply + cdi += 1 + # END handle rest + else: + for dc in dcl: + delta_chunk_apply(dc, bbuf, write) + # END for each dc + # END handle application values + class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks""" @@ -211,6 +266,11 @@ def connect_with(self, bdcl): # END handle overlap # END for each dc + def apply(self, bbuf, write, lbound_offset=0, size=0): + """Only used by public clients, internally we only use the global routines + for performance""" + return delta_list_apply(self, bbuf, write, lbound_offset, size) + def compress(self): """Alter the list to reduce the amount of nodes. Currently we concatenate add-chunks @@ -255,61 +315,6 @@ def compress(self): # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) return self - def apply(self, bbuf, write, lbound_offset=0, size=0): - """Apply the chain's changes and write the final result using the passed - write function. - :param bbuf: base buffer containing the base of all deltas contained in this - list. It will only be used if the chunk in question does not have a base - chain. - :param lbound_offset: offset at which to start applying the delta, relative to - our lbound - :param size: if larger than 0, only the given amount of bytes will be applied - :param write: function taking a string of bytes to write to the output""" - slen = len(self) - if slen == 0: - return - # END early abort - absofs = self.lbound() + lbound_offset - if size == 0: - size = self.rbound() - absofs - # END initialize size - - dapply = DeltaChunk.apply - if lbound_offset or absofs + size != self.rbound(): - cdi = _closest_index(self, absofs) - cd = self[cdi] - if cd.to != absofs: - tcd = delta_duplicate(cd) - _move_delta_lbound(tcd, absofs - cd.to) - _set_delta_rbound(tcd, min(tcd.ts, size)) - dapply(tcd, bbuf, write) - size -= tcd.ts - cdi += 1 - # END handle first chunk - - # here we have to either apply full chunks, or smaller ones, but - # we always start at the chunks target offset - while cdi < slen and size: - cd = self[cdi] - if cd.ts <= size: - dapply(cd, bbuf, write) - size -= cd.ts - else: - tcd = delta_duplicate(cd) - _set_delta_rbound(tcd, size) - dapply(tcd, bbuf, write) - size -= tcd.ts - break - # END handle bytes to apply - cdi += 1 - # END handle rest - assert size == 0 - else: - for dc in self: - dapply(dc, bbuf, write) - # END for each dc - # END handle application values - def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches target_size @@ -380,7 +385,6 @@ def __getslice__(self, absofs, size): # END hadle size cdi += 1 # END for each chunk - assert size == 0, "size was %i" % size # ndcl.check_integrity() return ndcl From e814fda2ef0b2de172fe06c2cbd1bc539a368262 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 12:04:59 +0200 Subject: [PATCH 0070/1392] First frame to implement the actual data aggregation, but ... its probbaly going to change quite a lot again --- fun.py | 46 +++++++++++++++++++++++++++++++--------------- stream.py | 2 +- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/fun.py b/fun.py index 253e64092..ec3662fe4 100644 --- a/fun.py +++ b/fun.py @@ -83,7 +83,7 @@ def _move_delta_lbound(d, bytes): return d def delta_duplicate(src): - return DeltaChunk(src.to, src.ts, src.so, src.data) + return DeltaChunk(src.to, src.ts, src.so, src.data, src.flags) def delta_chunk_apply(dc, bbuf, write): """Apply own data to the target buffer @@ -114,16 +114,18 @@ class DeltaChunk(object): 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None + 'flags' # currently only True or False ) - def __init__(self, to, ts, so, data): + def __init__(self, to, ts, so, data, flags): self.to = to self.ts = ts self.so = so self.data = data + self.flags = flags def __repr__(self): - return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") + return "DeltaChunk(%i, %i, %s, %s, %i)" % (self.to, self.ts, self.so, self.data or "", self.flags) #{ Interface @@ -253,18 +255,24 @@ def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - def connect_with(self, bdcl): + def connect_with(self, bdcl, tdcl): """Connect this instance's delta chunks virtually with the given base. This means that all copy deltas will simply apply to the given region of the given base. Afterwards, the base is optimized so that add-deltas will be truncated to the region actually used, or removed completely where adequate. This way, memory usage is reduced. - :param bdcl: DeltaChunkList to serve as base""" - for dc in self: - if not dc.has_data(): - dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) - # END handle overlap - # END for each dc + :param bdcl: DeltaChunkList to serve as base + :param tdcl: topmost delta chunk list. If set, reverse order is assumed + and the list is connected more efficiently""" + if tdcl is None: + for dc in self: + if not dc.has_data(): + dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) + # END handle overlap + # END for each dc + else: + raise NotImplementedError("todo") + # END handle order def apply(self, bbuf, write, lbound_offset=0, size=0): """Only used by public clients, internally we only use the global routines @@ -297,7 +305,7 @@ def compress(self): del(self[first_data_index:i-1]) buf = nd.getvalue() - self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf, False)) slen = len(self) i = first_data_index + 1 @@ -522,14 +530,18 @@ def reverse_connect_deltas(dcl, dstreams): :return: None""" raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") -def connect_deltas(dstreams): +def connect_deltas(dstreams, reverse): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks :param dstreams: iterable of delta stream objects. They must be ordered latest last, hence the delta to be applied last comes last, its oldest ancestor first + :param reverse: If False, the given iterable of delta-streams returns + items in from latest ancestor to the last delta. + If True, deltas are ordered so that the one to be applied last comes first. :return: DeltaChunkList, containing all operations to apply""" bdcl = None # data chunk list for initial base dcl = DeltaChunkList() + tdcl = None # topmost dcl, only effective if reverse is True for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() @@ -576,12 +588,12 @@ def connect_deltas(dstreams): rbound > base_size): break - dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None, False)) tbw += cp_size elif c: # NOTE: in C, the data chunks should probably be concatenated here. # In python, we do it as a post-process - dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) + dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c], False)) i += c tbw += c else: @@ -591,9 +603,13 @@ def connect_deltas(dstreams): dcl.compress() + if reverse and tdcl is None: + tdcl = dcl + # END handle reverse + # merge the lists ! if bdcl is not None: - dcl.connect_with(bdcl) + dcl.connect_with(bdcl, tdcl) # END handle merge # dcl.check_integrity() diff --git a/stream.py b/stream.py index 16c917a97..40a0c6c6a 100644 --- a/stream.py +++ b/stream.py @@ -338,7 +338,7 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = connect_deltas(reversed(self._dstreams)) + dcl = connect_deltas(self._dstreams, reverse=True) if len(dcl) == 0: self._size = 0 From 2e19424354d03a5351ac374dc8dd36f6b65b0d5e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 12:33:14 +0200 Subject: [PATCH 0071/1392] New Frame uses a distinct type to express the different mode of operation. This is clean enough to get going --- fun.py | 85 ++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 23 deletions(-) diff --git a/fun.py b/fun.py index ec3662fe4..05925721c 100644 --- a/fun.py +++ b/fun.py @@ -237,7 +237,13 @@ def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): class DeltaChunkList(list): - """List with special functionality to deal with DeltaChunks""" + """List with special functionality to deal with DeltaChunks. + There are two types of lists we represent. The one was created bottom-up, working + towards the latest delta, the other kind was created top-down, working from the + latest delta down to the earliest ancestor. This attribute is queryable + after all processing with is_reversed.""" + + __slots__ = tuple() def rbound(self): """:return: rightmost extend in bytes, absolute""" @@ -255,24 +261,18 @@ def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - def connect_with(self, bdcl, tdcl): + def connect_with(self, bdcl): """Connect this instance's delta chunks virtually with the given base. This means that all copy deltas will simply apply to the given region of the given base. Afterwards, the base is optimized so that add-deltas will be truncated to the region actually used, or removed completely where adequate. This way, memory usage is reduced. - :param bdcl: DeltaChunkList to serve as base - :param tdcl: topmost delta chunk list. If set, reverse order is assumed - and the list is connected more efficiently""" - if tdcl is None: - for dc in self: - if not dc.has_data(): - dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) - # END handle overlap - # END for each dc - else: - raise NotImplementedError("todo") - # END handle order + :param bdcl: DeltaChunkList to serve as base""" + for dc in self: + if not dc.has_data(): + dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) + # END handle overlap + # END for each dc def apply(self, bbuf, write, lbound_offset=0, size=0): """Only used by public clients, internally we only use the global routines @@ -396,8 +396,37 @@ def __getslice__(self, absofs, size): # ndcl.check_integrity() return ndcl - - + + +class TopdownDeltaChunkList(DeltaChunkList): + """Represents a list which is generated by feeding its ancestor streams one by + one""" + __slots__ = ('frozen', ) # if True, the list is frozen and can reproduce all data + # Will only be set in lists which where processed top-down + + def __init__(self): + self.frozen = False + + def connect_with_next_base(self, bdcl): + """Connect this chain with the next level of our base delta chunklist. + The goal in this game is to mark as many of our chunks rigid, hence they + cannot be changed by any of the upcoming bases anymore. Once all our + chunks are marked like that, we can stop all processing + :param bdcl: data chunk list being one of our bases. They must be fed in + consequtively and in order, towards the earliest ancestor delta + :return: True if processing was done. Use it to abort processing of + remaining streams""" + if self.frozen == 1: + # Can that ever be hit ? + return False + # END early abort + # mark us so that the is_reversed method returns True, without us thinking + # we are frozen + self.frozen = -1 + + raise NotImplementedError("todo") + return True + #} END structures @@ -540,8 +569,14 @@ def connect_deltas(dstreams, reverse): If True, deltas are ordered so that the one to be applied last comes first. :return: DeltaChunkList, containing all operations to apply""" bdcl = None # data chunk list for initial base - dcl = DeltaChunkList() tdcl = None # topmost dcl, only effective if reverse is True + + if reverse: + dcl = tdcl = TopdownDeltaChunkList() + else: + dcl = DeltaChunkList() + # END handle type of first chunk list + for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() @@ -603,13 +638,14 @@ def connect_deltas(dstreams, reverse): dcl.compress() - if reverse and tdcl is None: - tdcl = dcl - # END handle reverse - # merge the lists ! if bdcl is not None: - dcl.connect_with(bdcl, tdcl) + if tdcl: + if not tdcl.connect_with_next_base(dcl): + break + # END early abort + else: + dcl.connect_with(bdcl) # END handle merge # dcl.check_integrity() @@ -619,7 +655,10 @@ def connect_deltas(dstreams, reverse): dcl = DeltaChunkList() # END for each delta stream - return bdcl + if tdcl: + return tdcl + else: + return bdcl def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): From f6bd67ce92257c9b5191b58de960cedda5159778 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 14:56:07 +0200 Subject: [PATCH 0072/1392] Reverse delta aggregration appears to be working --- fun.py | 155 +++++++++++++++++++++++++++++++--------------- test/test_pack.py | 2 +- 2 files changed, 107 insertions(+), 50 deletions(-) diff --git a/fun.py b/fun.py index 05925721c..9e4671a09 100644 --- a/fun.py +++ b/fun.py @@ -235,6 +235,46 @@ def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): # END for each dc # END handle application values +def delta_list_slice(dcl, absofs, size): + """:return: Subsection of this list at the given absolute offset, with the given + size in bytes. + :return: DeltaChunkList (copy) which represents the given chunk""" + if len(dcl) == 0: + return DeltaChunkList() + + absofs = max(absofs, dcl.lbound()) + size = min(dcl.rbound() - dcl.lbound(), size) + cdi = _closest_index(dcl, absofs) # delta start index + cd = dcl[cdi] + slen = len(dcl) + ndcl = dcl.__class__() + + if cd.to != absofs: + tcd = delta_duplicate(cd) + _move_delta_lbound(tcd, absofs - cd.to) + _set_delta_rbound(tcd, min(tcd.ts, size)) + ndcl.append(tcd) + size -= tcd.ts + cdi += 1 + # END lbound overlap handling + + while cdi < slen and size: + # are we larger than the current block + cd = dcl[cdi] + if cd.ts <= size: + ndcl.append(delta_duplicate(cd)) + size -= cd.ts + else: + tcd = delta_duplicate(cd) + _set_delta_rbound(tcd, size) + ndcl.append(tcd) + size -= tcd.ts + break + # END hadle size + cdi += 1 + # END for each chunk + + return ndcl class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks. @@ -270,7 +310,7 @@ def connect_with(self, bdcl): :param bdcl: DeltaChunkList to serve as base""" for dc in self: if not dc.has_data(): - dc.set_copy_chunklist(bdcl[dc.so:dc.ts]) + dc.set_copy_chunklist(delta_list_slice(bdcl, dc.so, dc.ts)) # END handle overlap # END for each dc @@ -355,49 +395,7 @@ def check_integrity(self, target_size=-1): assert lft.to + lft.ts == rgt.to # END for each pair - def __getslice__(self, absofs, size): - """:return: Subsection of this list at the given absolute offset, with the given - size in bytes. - :return: DeltaChunkList (copy) which represents the given chunk""" - if len(self) == 0: - return DeltaChunkList() - - absofs = max(absofs, self.lbound()) - size = min(self.rbound() - self.lbound(), size) - cdi = _closest_index(self, absofs) # delta start index - cd = self[cdi] - slen = len(self) - ndcl = self.__class__() - - if cd.to != absofs: - tcd = delta_duplicate(cd) - _move_delta_lbound(tcd, absofs - cd.to) - _set_delta_rbound(tcd, min(tcd.ts, size)) - ndcl.append(tcd) - size -= tcd.ts - cdi += 1 - # END lbound overlap handling - - while cdi < slen and size: - # are we larger than the current block - cd = self[cdi] - if cd.ts <= size: - ndcl.append(delta_duplicate(cd)) - size -= cd.ts - else: - tcd = delta_duplicate(cd) - _set_delta_rbound(tcd, size) - ndcl.append(tcd) - size -= tcd.ts - break - # END hadle size - cdi += 1 - # END for each chunk - - # ndcl.check_integrity() - return ndcl - class TopdownDeltaChunkList(DeltaChunkList): """Represents a list which is generated by feeding its ancestor streams one by one""" @@ -416,15 +414,76 @@ def connect_with_next_base(self, bdcl): consequtively and in order, towards the earliest ancestor delta :return: True if processing was done. Use it to abort processing of remaining streams""" + assert self is not bdcl if self.frozen == 1: # Can that ever be hit ? return False # END early abort - # mark us so that the is_reversed method returns True, without us thinking - # we are frozen - self.frozen = -1 - raise NotImplementedError("todo") + nfc = 0 # number of frozen chunks + dci = 0 # delta chunk index + slen = len(self) # len of self + sold = slen + while dci < slen: + dc = self[dci] + dci += 1 + + if dc.flags: + nfc += 1 + continue + # END skip frozen chunks + + # all data chunks must be frozen, we are topmost already + # (Also if its a copy operation onto the lowest base, but we cannot + # determine that without the number of deltas to come) + if dc.has_data(): + dc.flags = True + nfc += 1 + continue + # END skip add chunks + + # copy chunks + # integrate the portion of the base list into ourselves. Lists + # dont support efficient insertion ( just one at a time ), but for now + # we live with it. Internally, its all just a 32/64bit pointer, and + # the portions of moved memory should be smallish. Maybe we just rebuild + # ourselves in order to reduce the amount of insertions ... + ccl = delta_list_slice(bdcl, dc.so, dc.ts) + + # move the target bounds into place to match with our chunk + ofs = dc.to - dc.so + for cdc in ccl: + cdc.to += ofs + # END update target bounds + + + assert dc.to == ccl.lbound() and dc.rbound() == cdc.rbound() + + if len(ccl) == 1: + self[dci-1] = ccl[0] + else: + + # maybe try to compute the expenses here, and pick the right algorithm + # It would normally be faster than copying everything physically though + # TODO: Use a deque here, and decide by the index whether to extend + # or extend left ! + post_dci = self[dci:] + del(self[dci-1:]) # include deletion of dc + self.extend(ccl) + self.extend(post_dci) + + slen = len(self) + dci += len(ccl)-1 # deleted dc, added rest + + # END handle chunk replacement + + # END for each chunk + + if nfc == slen: + self.frozen = True + return False + # END handle completeness + return True @@ -648,8 +707,6 @@ def connect_deltas(dstreams, reverse): dcl.connect_with(bdcl) # END handle merge - # dcl.check_integrity() - # prepare next base bdcl = dcl dcl = DeltaChunkList() diff --git a/test/test_pack.py b/test/test_pack.py index 6e598d755..770a78bad 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -130,7 +130,7 @@ def test_pack(self): self._assert_pack_file(pack, version, size) # END for each pack to test - def _test_pack_entity(self): + def test_pack_entity(self): for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): From 0381cae63284e237a7023e9d40c7772690a2f84e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 15:11:02 +0200 Subject: [PATCH 0073/1392] Removed debugging code --- fun.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/fun.py b/fun.py index 9e4671a09..814c22a6a 100644 --- a/fun.py +++ b/fun.py @@ -399,11 +399,7 @@ def check_integrity(self, target_size=-1): class TopdownDeltaChunkList(DeltaChunkList): """Represents a list which is generated by feeding its ancestor streams one by one""" - __slots__ = ('frozen', ) # if True, the list is frozen and can reproduce all data - # Will only be set in lists which where processed top-down - - def __init__(self): - self.frozen = False + __slots__ = tuple() def connect_with_next_base(self, bdcl): """Connect this chain with the next level of our base delta chunklist. @@ -413,17 +409,10 @@ def connect_with_next_base(self, bdcl): :param bdcl: data chunk list being one of our bases. They must be fed in consequtively and in order, towards the earliest ancestor delta :return: True if processing was done. Use it to abort processing of - remaining streams""" - assert self is not bdcl - if self.frozen == 1: - # Can that ever be hit ? - return False - # END early abort - + remaining streams if False is returned""" nfc = 0 # number of frozen chunks dci = 0 # delta chunk index slen = len(self) # len of self - sold = slen while dci < slen: dc = self[dci] dci += 1 @@ -456,9 +445,6 @@ def connect_with_next_base(self, bdcl): cdc.to += ofs # END update target bounds - - assert dc.to == ccl.lbound() and dc.rbound() == cdc.rbound() - if len(ccl) == 1: self[dci-1] = ccl[0] else: @@ -476,14 +462,11 @@ def connect_with_next_base(self, bdcl): dci += len(ccl)-1 # deleted dc, added rest # END handle chunk replacement - # END for each chunk if nfc == slen: - self.frozen = True return False # END handle completeness - return True From 9b0773b92df4b4a2a53497efc9c1489028d403d7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 15:32:14 +0200 Subject: [PATCH 0074/1392] Removed previous non-reverse delta-application functionality. Although it was slightly faster, this new version only needs a faster slicing, which consumes ridiculous amounts of time --- fun.py | 134 ++++++++++++------------------------------------------ stream.py | 9 ++-- 2 files changed, 32 insertions(+), 111 deletions(-) diff --git a/fun.py b/fun.py index 814c22a6a..4d5edda9c 100644 --- a/fun.py +++ b/fun.py @@ -62,7 +62,6 @@ def _set_delta_rbound(d, size): # NOTE: data is truncated automatically when applying the delta # MUST NOT DO THIS HERE - return d def _move_delta_lbound(d, bytes): @@ -76,14 +75,14 @@ def _move_delta_lbound(d, bytes): d.to += bytes d.so += bytes d.ts -= bytes - if d.has_data(): + if d.data is not None: d.data = d.data[bytes:] # END handle data return d def delta_duplicate(src): - return DeltaChunk(src.to, src.ts, src.so, src.data, src.flags) + return DeltaChunk(src.to, src.ts, src.so, src.data) def delta_chunk_apply(dc, bbuf, write): """Apply own data to the target buffer @@ -92,8 +91,6 @@ def delta_chunk_apply(dc, bbuf, write): if dc.data is None: # COPY DATA FROM SOURCE write(buffer(bbuf, dc.so, dc.ts)) - elif isinstance(dc.data, DeltaChunkList): - delta_list_apply(dc.data, bbuf, write, dc.so, dc.ts) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? @@ -105,6 +102,7 @@ def delta_chunk_apply(dc, bbuf, write): # END handle truncation # END handle chunk mode + class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" @@ -114,51 +112,25 @@ class DeltaChunk(object): 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None - 'flags' # currently only True or False ) - def __init__(self, to, ts, so, data, flags): + def __init__(self, to, ts, so, data): self.to = to self.ts = ts self.so = so self.data = data - self.flags = flags def __repr__(self): - return "DeltaChunk(%i, %i, %s, %s, %i)" % (self.to, self.ts, self.so, self.data or "", self.flags) + return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") #{ Interface - def copy_offset(self): - """:return: offset to apply when copying from a base buffer, or 0 - if this is not a copying delta chunk""" - - if self.data is not None: - if isinstance(self.data, DeltaChunkList): - return self.data.lbound() + self.so - else: - return self.so - # END handle data type - return 0 - def rbound(self): return self.to + self.ts def has_data(self): """:return: True if the instance has data to add to the target stream""" - return self.data is not None and not isinstance(self.data, DeltaChunkList) - - def has_copy_chunklist(self): - """:return: True if we copy our data from a chunklist""" - return self.data is not None and isinstance(self.data, DeltaChunkList) - - def set_copy_chunklist(self, dcl): - """Set the deltachunk list to be used as basis for copying. - :note: only works if this chunk is a copy delta chunk""" - self.data = dcl - self.so = 0 # allows lbound moves to be virtual - - + return self.data is not None #} END interface @@ -239,21 +211,20 @@ def delta_list_slice(dcl, absofs, size): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. :return: DeltaChunkList (copy) which represents the given chunk""" - if len(dcl) == 0: - return DeltaChunkList() - - absofs = max(absofs, dcl.lbound()) - size = min(dcl.rbound() - dcl.lbound(), size) + dcllbound = dcl.lbound() + absofs = max(absofs, dcllbound) + size = min(dcl.rbound() - dcllbound, size) cdi = _closest_index(dcl, absofs) # delta start index cd = dcl[cdi] slen = len(dcl) - ndcl = dcl.__class__() + ndcl = DeltaChunkList() + lappend = ndcl.append if cd.to != absofs: tcd = delta_duplicate(cd) _move_delta_lbound(tcd, absofs - cd.to) _set_delta_rbound(tcd, min(tcd.ts, size)) - ndcl.append(tcd) + lappend(tcd) size -= tcd.ts cdi += 1 # END lbound overlap handling @@ -262,12 +233,12 @@ def delta_list_slice(dcl, absofs, size): # are we larger than the current block cd = dcl[cdi] if cd.ts <= size: - ndcl.append(delta_duplicate(cd)) + lappend(delta_duplicate(cd)) size -= cd.ts else: tcd = delta_duplicate(cd) _set_delta_rbound(tcd, size) - ndcl.append(tcd) + lappend(tcd) size -= tcd.ts break # END hadle size @@ -301,19 +272,6 @@ def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - def connect_with(self, bdcl): - """Connect this instance's delta chunks virtually with the given base. - This means that all copy deltas will simply apply to the given region - of the given base. Afterwards, the base is optimized so that add-deltas - will be truncated to the region actually used, or removed completely where - adequate. This way, memory usage is reduced. - :param bdcl: DeltaChunkList to serve as base""" - for dc in self: - if not dc.has_data(): - dc.set_copy_chunklist(delta_list_slice(bdcl, dc.so, dc.ts)) - # END handle overlap - # END for each dc - def apply(self, bbuf, write, lbound_offset=0, size=0): """Only used by public clients, internally we only use the global routines for performance""" @@ -333,7 +291,7 @@ def compress(self): while i < slen: dc = self[i] i += 1 - if not dc.has_data(): + if dc.data is None: if first_data_index is not None and i-2-first_data_index > 1: #if first_data_index is not None: nd = StringIO() # new data @@ -345,7 +303,7 @@ def compress(self): del(self[first_data_index:i-1]) buf = nd.getvalue() - self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf, False)) + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) slen = len(self) i = first_data_index + 1 @@ -381,8 +339,6 @@ def check_integrity(self, target_size=-1): assert dc.ts > 0 if dc.has_data(): assert len(dc.data) >= dc.ts - if dc.has_copy_chunklist(): - assert dc.ts <= dc.data.size() # END for each dc left = islice(self, 0, len(self)-1) @@ -417,16 +373,8 @@ def connect_with_next_base(self, bdcl): dc = self[dci] dci += 1 - if dc.flags: - nfc += 1 - continue - # END skip frozen chunks - - # all data chunks must be frozen, we are topmost already - # (Also if its a copy operation onto the lowest base, but we cannot - # determine that without the number of deltas to come) - if dc.has_data(): - dc.flags = True + # all add-chunks which are already topmost don't need additional processing + if dc.data is not None: nfc += 1 continue # END skip add chunks @@ -448,7 +396,6 @@ def connect_with_next_base(self, bdcl): if len(ccl) == 1: self[dci-1] = ccl[0] else: - # maybe try to compute the expenses here, and pick the right algorithm # It would normally be faster than copying everything physically though # TODO: Use a deque here, and decide by the index whether to extend @@ -592,33 +539,16 @@ def stream_copy(read, write, size, chunk_size): # END duplicate data return dbw -def reverse_connect_deltas(dcl, dstreams): +def connect_deltas(dstreams): """Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - :param dcl: see 3 - :param dstreams: iterable of delta stream objects. They must be ordered latest first, - hence the delta to be applied last comes first, then its ancestors - :return: None""" - raise NotImplementedError("This is left out up until we actually iterate the dstreams - they are prefetched right now") - -def connect_deltas(dstreams, reverse): - """Read the condensed delta chunk information from dstream and merge its information - into a list of existing delta chunks - :param dstreams: iterable of delta stream objects. They must be ordered latest last, - hence the delta to be applied last comes last, its oldest ancestor first - :param reverse: If False, the given iterable of delta-streams returns - items in from latest ancestor to the last delta. - If True, deltas are ordered so that the one to be applied last comes first. + :param dstreams: iterable of delta stream objects, the delta to be applied last + comes first, then all its ancestors in order :return: DeltaChunkList, containing all operations to apply""" bdcl = None # data chunk list for initial base - tdcl = None # topmost dcl, only effective if reverse is True - - if reverse: - dcl = tdcl = TopdownDeltaChunkList() - else: - dcl = DeltaChunkList() - # END handle type of first chunk list + tdcl = None # topmost dcl + dcl = tdcl = TopdownDeltaChunkList() for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() @@ -665,12 +595,12 @@ def connect_deltas(dstreams, reverse): rbound > base_size): break - dcl.append(DeltaChunk(tbw, cp_size, cp_off, None, False)) + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: # NOTE: in C, the data chunks should probably be concatenated here. # In python, we do it as a post-process - dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c], False)) + dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) i += c tbw += c else: @@ -682,12 +612,8 @@ def connect_deltas(dstreams, reverse): # merge the lists ! if bdcl is not None: - if tdcl: - if not tdcl.connect_with_next_base(dcl): - break - # END early abort - else: - dcl.connect_with(bdcl) + if not tdcl.connect_with_next_base(dcl): + break # END handle merge # prepare next base @@ -695,11 +621,7 @@ def connect_deltas(dstreams, reverse): dcl = DeltaChunkList() # END for each delta stream - if tdcl: - return tdcl - else: - return bdcl - + return tdcl def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ diff --git a/stream.py b/stream.py index 40a0c6c6a..5292ce512 100644 --- a/stream.py +++ b/stream.py @@ -328,17 +328,16 @@ def __init__(self, stream_list): def _set_cache_(self, attr): # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller - # than X - definitely the case in python - # hence we apply a worst-case scenario here - # TODO: read the final size from the deltastream - have to partly unpack - # if len(self._dstreams) * self._size < self.k_max_memory_move: + # than X - definitely the case in python, every function call costs + # huge amounts of time + # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: if len(self._dstreams) == 1: return self._set_cache_brute_(attr) # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - dcl = connect_deltas(self._dstreams, reverse=True) + dcl = connect_deltas(self._dstreams) if len(dcl) == 0: self._size = 0 From 3837806673b992c1c0cb64203b50d9f1054e44db Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 15:45:54 +0200 Subject: [PATCH 0075/1392] Improved performance of delta_chunk_slice method a tiny bit, but it really needs to go to c --- fun.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fun.py b/fun.py index 4d5edda9c..75ff800be 100644 --- a/fun.py +++ b/fun.py @@ -210,14 +210,14 @@ def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): def delta_list_slice(dcl, absofs, size): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. - :return: DeltaChunkList (copy) which represents the given chunk""" + :return: list (copy) which represents the given chunk""" dcllbound = dcl.lbound() absofs = max(absofs, dcllbound) size = min(dcl.rbound() - dcllbound, size) cdi = _closest_index(dcl, absofs) # delta start index cd = dcl[cdi] slen = len(dcl) - ndcl = DeltaChunkList() + ndcl = list() lappend = ndcl.append if cd.to != absofs: From 89408f8ec351c1d14f66caf53a2c1e163bde0f27 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 11 Oct 2010 23:08:57 +0200 Subject: [PATCH 0076/1392] Initial frame of the connect_delta method, which seems to do something. Debugging is hellish, you really have to use python exception to get information out of there, printf doesn't do anything for some reason --- Makefile | 24 +++++ _fun.c | 279 +++++++++++++++++++++++++++++++++++++++++++++++++++++- fun.py | 5 + stream.py | 5 +- 4 files changed, 310 insertions(+), 3 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..190a66b03 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +PYTHON = python +SETUP = $(PYTHON) setup.py +TESTRUNNER = $(shell which nosetests) +TESTFLAGS = + +all: build + +doc:: + make -C doc/ html + +build:: + $(SETUP) build + $(SETUP) build_ext -i + +install:: + $(SETUP) install + +clean:: + $(SETUP) clean --all + rm -f *.so + +coverage:: build + PYTHONPATH=. $(PYTHON) $(TESTRUNNER) --cover-package=dulwich --with-coverage --cover-erase --cover-inclusive gitdb + diff --git a/_fun.c b/_fun.c index ce9f25b16..e9f769daa 100644 --- a/_fun.c +++ b/_fun.c @@ -1,5 +1,7 @@ #include #include +#include +#include static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -82,16 +84,289 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) } +typedef unsigned long long ull; + +// Internal Delta Chunk Objects +typedef struct { + ull to; + ull ts; + ull so; + PyObject* data; + + void* next; +} DeltaChunk; + + +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, PyObject* data, DeltaChunk* next) +{ + dc->to = to; + dc->ts = ts; + dc->so = so; + Py_XINCREF(data); + dc->data = data; + + dc->next = next; +} + +void DC_destroy(DeltaChunk* dc) +{ + Py_XDECREF(dc->data); +} + +typedef struct { + PyObject_HEAD + // ----------- + DeltaChunk* head; + DeltaChunk* tail; + ull size; + +} DeltaChunkList; + +ull DC_rbound(DeltaChunk* dc) +{ + return dc->to + dc->ts; +} + + +static +int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) +{ + ((DeltaChunkList*)self)->head = NULL; + return 1; +} + +static +void DCL_dealloc(DeltaChunkList* self) +{ + // TODO: deallocate linked list + if (self->head){ + self->head = NULL; + self->tail = NULL; + self->size = 0; + } +} + +static +PyObject* DCL_len(PyObject* self) +{ + return PyLong_FromUnsignedLongLong(0); +} + +static +PyObject* DCL_rbound(DeltaChunkList* self) +{ + if (!self->head) + return PyLong_FromUnsignedLongLong(0); + return PyLong_FromUnsignedLongLong(DC_rbound(self->tail)); +} + +static +PyObject* DCL_apply(PyObject* self, PyObject* args) +{ + + Py_RETURN_NONE; +} + + + +static PyMethodDef DCL_methods[] = { + {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, + {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, + {"rbound", (PyCFunction)DCL_rbound, METH_NOARGS, NULL}, + {NULL} /* Sentinel */ +}; + +static PyTypeObject DeltaChunkListType = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "DeltaChunkList", /*tp_name*/ + sizeof(DeltaChunkList), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)DCL_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + "Minimal Delta Chunk List",/* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + DCL_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)DCL_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + +static inline +ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* out_bytes_read){ + ull size = 0; + Py_ssize_t i = 0; + const char* dend = data + dlen; + for (data = data + offset; data < dend; data+=1, i+=1){ + char c = *data; + size |= (c & 0x7f) << i*7; + if (!(c & 0x80)){ + break; + } + }// END while in range + + *out_bytes_read = i+offset; + return size; +} + +static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) +{ + // obtain iterator + PyObject* stream_iter = 0; + if (!PyIter_Check(dstreams)){ + stream_iter = PyObject_GetIter(dstreams); + if (!stream_iter){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't obtain iterator for streams"); + return NULL; + } + } else { + stream_iter = dstreams; + } + + DeltaChunkList* bdcl = 0; + DeltaChunkList* tdcl = 0; + DeltaChunkList* dcl = 0; + + dcl = tdcl = PyObject_New(DeltaChunkList, &DeltaChunkListType); + if (!dcl){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); + return NULL; + } + + unsigned int dsi; + PyObject* ds; + int error = 0; + for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) + { + PyObject* db = PyObject_CallMethod(ds, "read", 0); + if (!PyObject_CheckReadBuffer(db)){ + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); + goto loop_end; + } + + const char* data; + Py_ssize_t dlen; + PyObject_AsReadBuffer(db, (const void**)&data, &dlen); + + // read header + Py_ssize_t ofs = 0; + const ull base_size = msb_size(data, dlen, 0, &ofs); + const ull target_size = msb_size(data, dlen, ofs, &ofs); + + // parse command stream + const char* dend = data + dlen; + ull tbw = 0; // Amount of target bytes written + for (data = data + ofs; data < dend; ++data) + { + const char cmd = *data; + + if (cmd & 0x80) + { + unsigned long cp_off = 0, cp_size = 0; + if (cmd & 0x01) cp_off = *data++; + if (cmd & 0x02) cp_off |= (*data++ << 8); + if (cmd & 0x04) cp_off |= (*data++ << 16); + if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); + if (cmd & 0x10) cp_size = *data++; + if (cmd & 0x20) cp_size |= (*data++ << 8); + if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cp_size == 0) cp_size = 0x10000; + + const unsigned long rbound = cp_off + cp_size; + if (rbound < cp_size || + rbound > base_size){ + goto loop_end; + } + + // TODO: Add node + tbw += cp_size; + + } else if (cmd) { + // TODO: Add node + tbw += cmd; + } else { + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + goto loop_end; + } + }// END handle command opcodes + + assert(tbw == target_size); + +loop_end: + // perform cleanup + Py_DECREF(ds); + Py_DECREF(db); + + if (error){ + break; + } + }// END for each stream object + + if (dsi == 0 && ! error){ + PyErr_SetString(PyExc_ValueError, "No streams provided"); + } + + if (stream_iter != dstreams){ + Py_DECREF(stream_iter); + } + + if (error){ + return NULL; + } + + return (PyObject*)tdcl; +} + static PyMethodDef py_fun[] = { - { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, NULL }, + { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, + { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, { NULL, NULL, 0, NULL } }; -void init_fun(void) +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif +PyMODINIT_FUNC init_fun(void) { PyObject *m; + DeltaChunkListType.tp_new = PyType_GenericNew; + if (PyType_Ready(&DeltaChunkListType) < 0) + return; + m = Py_InitModule3("_fun", py_fun, NULL); if (m == NULL) return; + + Py_INCREF(&DeltaChunkListType); + PyModule_AddObject(m, "Noddy", (PyObject *)&DeltaChunkListType); } diff --git a/fun.py b/fun.py index 75ff800be..a4da30903 100644 --- a/fun.py +++ b/fun.py @@ -701,3 +701,8 @@ def is_equal_canonical_sha(canonical_length, match, sha1): #} END routines + +try: + from _fun import connect_deltas +except ImportError: + pass diff --git a/stream.py b/stream.py index 5292ce512..169104625 100644 --- a/stream.py +++ b/stream.py @@ -338,8 +338,11 @@ def _set_cache_(self, attr): # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. dcl = connect_deltas(self._dstreams) + assert dcl is not None - if len(dcl) == 0: + # call len directly, as the (optional) c version doesn't implement the sequence + # protocol + if dcl.__len__() == 0: self._size = 0 self._mm_target = allocate_memory(0) return From 2409fafca6f0518500571bcf82e92554f2c50b85 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 09:19:12 +0200 Subject: [PATCH 0077/1392] Implemented a few more functions, but I realize the vector implementation actually wants to be in a separate structure --- _fun.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/_fun.c b/_fun.c index e9f769daa..8abc5bcbb 100644 --- a/_fun.c +++ b/_fun.c @@ -116,9 +116,9 @@ void DC_destroy(DeltaChunk* dc) typedef struct { PyObject_HEAD // ----------- - DeltaChunk* head; - DeltaChunk* tail; + DeltaChunk* mem; ull size; + ull reserved_size; } DeltaChunkList; @@ -127,22 +127,65 @@ ull DC_rbound(DeltaChunk* dc) return dc->to + dc->ts; } +static +int DCL_new(DeltaChunkList* self, PyObject* args, PyObject* kwds) +{ + self->mem = NULL; // Memory + self->size = 0; // Size in DeltaChunks + self->reserved_size = 0; // Reserve in DeltaChunks + return 1; +} + +/* +Grow the delta chunk list by the given amount of bytes. +This may trigger a realloc, but will do nothing if the reserved size is already +large enough. +Return 1 on success, 0 on failure +*/ +static +int DCL_grow(DeltaChunkList* self, ull num_dc) +{ + const ull grow_by_chunks = (self->size + num_dc) - self->reserved_size; + if (grow_by_chunks <= 0){ + return 1; + } + + if (self->mem){ + self->mem = PyMem_Malloc(grow_by_chunks*sizeof(DeltaChunk)); + } else { + self->mem = PyMem_Realloc(self->mem, (self->size + grow_by_chunks)*sizeof(DeltaChunk)); + } + + return self->mem != NULL; +} + static int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) { - ((DeltaChunkList*)self)->head = NULL; - return 1; + if(PySequence_Size(args) > 1){ + PyErr_SetString(PyExc_ValueError, "Zero or one arguments are allowed, providing the initial size of the queue in DeltaChunks"); + return 0; + } + + ull init_size = 0; + PyArg_ParseTuple(args, "K", &init_size); + if (init_size == 0){ + init_size = 125000; + } + + return DCL_grow(self, init_size); } static void DCL_dealloc(DeltaChunkList* self) { // TODO: deallocate linked list - if (self->head){ - self->head = NULL; - self->tail = NULL; + if (self->mem){ + PyMem_Free(self->mem); self->size = 0; + self->reserved_size = 0; + self->mem = 0; } } @@ -153,11 +196,18 @@ PyObject* DCL_len(PyObject* self) } static -PyObject* DCL_rbound(DeltaChunkList* self) +inline +ull DCL_rbound(DeltaChunkList* self) +{ + if (!self->mem | !self->size) + return 0; + return DC_rbound(&(self->mem[self->size-1])); +} + +static +PyObject* DCL_py_rbound(DeltaChunkList* self) { - if (!self->head) - return PyLong_FromUnsignedLongLong(0); - return PyLong_FromUnsignedLongLong(DC_rbound(self->tail)); + return PyLong_FromUnsignedLongLong(DCL_rbound(self)); } static @@ -172,7 +222,7 @@ PyObject* DCL_apply(PyObject* self, PyObject* args) static PyMethodDef DCL_methods[] = { {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, - {"rbound", (PyCFunction)DCL_rbound, METH_NOARGS, NULL}, + {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, {NULL} /* Sentinel */ }; @@ -215,7 +265,7 @@ static PyTypeObject DeltaChunkListType = { 0, /* tp_dictoffset */ (initproc)DCL_init, /* tp_init */ 0, /* tp_alloc */ - 0, /* tp_new */ + (newfunc)DCL_new, /* tp_new */ }; @@ -233,6 +283,7 @@ ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* o }// END while in range *out_bytes_read = i+offset; + assert((*out_bytes_read * 8) - (*out_bytes_read - 1) <= sizeof(ull)); return size; } From 511a29dab7c7a507abc3bc656b5e9f5ab2db85a3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 09:54:15 +0200 Subject: [PATCH 0078/1392] DeltaChunkVector is now a separate structure. I wished I had c++, but ... its probably a good exercise --- _fun.c | 126 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 83 insertions(+), 43 deletions(-) diff --git a/_fun.c b/_fun.c index 8abc5bcbb..fb83f26a6 100644 --- a/_fun.c +++ b/_fun.c @@ -85,27 +85,26 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) typedef unsigned long long ull; +typedef unsigned int uint; + +// DELTA CHUNK +//////////////// // Internal Delta Chunk Objects typedef struct { ull to; ull ts; ull so; PyObject* data; - - void* next; } DeltaChunk; - -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, PyObject* data, DeltaChunk* next) +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, PyObject* data) { dc->to = to; dc->ts = ts; dc->so = so; Py_XINCREF(data); dc->data = data; - - dc->next = next; } void DC_destroy(DeltaChunk* dc) @@ -113,28 +112,20 @@ void DC_destroy(DeltaChunk* dc) Py_XDECREF(dc->data); } -typedef struct { - PyObject_HEAD - // ----------- - DeltaChunk* mem; - ull size; - ull reserved_size; - -} DeltaChunkList; - ull DC_rbound(DeltaChunk* dc) { return dc->to + dc->ts; } -static -int DCL_new(DeltaChunkList* self, PyObject* args, PyObject* kwds) -{ - self->mem = NULL; // Memory - self->size = 0; // Size in DeltaChunks - self->reserved_size = 0; // Reserve in DeltaChunks - return 1; -} + +// DELTA CHUNK VECTOR +///////////////////// + +typedef struct { + DeltaChunk* mem; // Memory + Py_ssize_t size; // Size in DeltaChunks + Py_ssize_t reserved_size; // Reserve in DeltaChunks +} DeltaChunkVector; /* Grow the delta chunk list by the given amount of bytes. @@ -143,20 +134,76 @@ large enough. Return 1 on success, 0 on failure */ static -int DCL_grow(DeltaChunkList* self, ull num_dc) +int DCV_grow(DeltaChunkVector* vec, uint num_dc) { - const ull grow_by_chunks = (self->size + num_dc) - self->reserved_size; + const ull grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; if (grow_by_chunks <= 0){ return 1; } - if (self->mem){ - self->mem = PyMem_Malloc(grow_by_chunks*sizeof(DeltaChunk)); + if (vec->mem){ + vec->mem = PyMem_Malloc(grow_by_chunks*sizeof(DeltaChunk)); } else { - self->mem = PyMem_Realloc(self->mem, (self->size + grow_by_chunks)*sizeof(DeltaChunk)); + vec->mem = PyMem_Realloc(vec->mem, (vec->size + grow_by_chunks)*sizeof(DeltaChunk)); } - return self->mem != NULL; + return vec->mem != NULL; +} + +int DCV_init(DeltaChunkVector* vec, ull initial_size) +{ + vec->mem = NULL; + vec->size = 0; + vec->reserved_size = 0; + + return DCV_grow(vec, initial_size); +} + + +void DCV_dealloc(DeltaChunkVector* vec) +{ + if (vec->mem){ + PyMem_Free(vec->mem); + vec->size = 0; + vec->reserved_size = 0; + vec->mem = 0; + } +} + +static inline +ull DCV_len(DeltaChunkVector* vec) +{ + return vec->size; +} + +// Return item at index +static inline +DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) +{ + assert(i < vec->size && vec->mem); + return &(vec->mem[i]); +} + +static inline +int DCV_empty(DeltaChunkVector* vec) +{ + return vec->size == 0; +} + +// DELTA CHUNK LIST (PYTHON) +///////////////////////////// + +typedef struct { + PyObject_HEAD + // ----------- + DeltaChunkVector vec; + +} DeltaChunkList; + +static +int DCL_new(DeltaChunkList* self, PyObject* args, PyObject* kwds) +{ + return DCV_init(&self->vec, 0); } @@ -174,34 +221,27 @@ int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) init_size = 125000; } - return DCL_grow(self, init_size); + return DCV_grow(&self->vec, init_size); } static void DCL_dealloc(DeltaChunkList* self) { - // TODO: deallocate linked list - if (self->mem){ - PyMem_Free(self->mem); - self->size = 0; - self->reserved_size = 0; - self->mem = 0; - } + DCV_dealloc(&self->vec); } static -PyObject* DCL_len(PyObject* self) +PyObject* DCL_len(DeltaChunkList* self) { - return PyLong_FromUnsignedLongLong(0); + return PyLong_FromUnsignedLongLong(DCV_len(&self->vec)); } -static -inline +static inline ull DCL_rbound(DeltaChunkList* self) { - if (!self->mem | !self->size) + if (DCV_empty(&self->vec)) return 0; - return DC_rbound(&(self->mem[self->size-1])); + return DC_rbound(DCV_get(&self->vec, self->vec.size - 1)); } static From f030fa1d7ef6a43cc05b11e3be2108f83d9683f9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 11:25:23 +0200 Subject: [PATCH 0079/1392] Weird bug causes crash, its memory related of course. GDB tells me where, but the why is still a mystery --- _fun.c | 82 +++++++++++++++++++++++++++++++++++++------------------ stream.py | 3 ++ 2 files changed, 58 insertions(+), 27 deletions(-) diff --git a/_fun.c b/_fun.c index fb83f26a6..47f5d8ecf 100644 --- a/_fun.c +++ b/_fun.c @@ -136,16 +136,18 @@ Return 1 on success, 0 on failure static int DCV_grow(DeltaChunkVector* vec, uint num_dc) { - const ull grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; + const uint grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; if (grow_by_chunks <= 0){ return 1; } - if (vec->mem){ - vec->mem = PyMem_Malloc(grow_by_chunks*sizeof(DeltaChunk)); + if (vec->mem == NULL){ + vec->mem = PyMem_Malloc(grow_by_chunks * sizeof(vec->mem)); } else { - vec->mem = PyMem_Realloc(vec->mem, (vec->size + grow_by_chunks)*sizeof(DeltaChunk)); + vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(vec->mem)); } + assert(vec->mem != NULL); + vec->reserved_size = vec->reserved_size + grow_by_chunks; return vec->mem != NULL; } @@ -159,17 +161,6 @@ int DCV_init(DeltaChunkVector* vec, ull initial_size) return DCV_grow(vec, initial_size); } - -void DCV_dealloc(DeltaChunkVector* vec) -{ - if (vec->mem){ - PyMem_Free(vec->mem); - vec->size = 0; - vec->reserved_size = 0; - vec->mem = 0; - } -} - static inline ull DCV_len(DeltaChunkVector* vec) { @@ -181,7 +172,7 @@ static inline DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) { assert(i < vec->size && vec->mem); - return &(vec->mem[i]); + return &vec->mem[i]; } static inline @@ -190,6 +181,48 @@ int DCV_empty(DeltaChunkVector* vec) return vec->size == 0; } +// Return end pointer of the vector +static inline +DeltaChunk* DCV_end(DeltaChunkVector* vec) +{ + assert(!DCV_empty(vec)); + return &vec->mem[vec->size]; +} + +void DCV_dealloc(DeltaChunkVector* vec) +{ + if (vec->mem){ + if (vec->size){ + const DeltaChunk* end = DCV_end(vec); + DeltaChunk* i; + for(i = &vec->mem[0]; i < end; i++){ + DC_destroy(i); + } + } + PyMem_Free(vec->mem); + vec->size = 0; + vec->reserved_size = 0; + vec->mem = 0; + } +} + +// Append num-chunks to the end of the list, possibly reallocating existing ones +// Return a pointer to the first of the added items. They are not yet initialized +// If num-chunks == 0, it returns the end pointer of the allocated memory +static inline +DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) +{ + if (vec->size + num_chunks > vec->reserved_size){ + if (!DCV_grow(vec, (vec->size + num_chunks) - vec->reserved_size)){ + Py_FatalError("Could not allocate memory for append operation"); + } + } + Py_FatalError("Could not allocate memory for append operation"); + Py_ssize_t old_size = vec->size; + vec->size += num_chunks; + return &vec->mem[old_size]; +} + // DELTA CHUNK LIST (PYTHON) ///////////////////////////// @@ -200,12 +233,6 @@ typedef struct { } DeltaChunkList; -static -int DCL_new(DeltaChunkList* self, PyObject* args, PyObject* kwds) -{ - return DCV_init(&self->vec, 0); -} - static int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) @@ -215,19 +242,20 @@ int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) return 0; } + assert(self->vec.mem == NULL); + ull init_size = 0; PyArg_ParseTuple(args, "K", &init_size); if (init_size == 0){ - init_size = 125000; + init_size = 12500; } - - return DCV_grow(&self->vec, init_size); + return DCV_init(&self->vec, init_size); } static void DCL_dealloc(DeltaChunkList* self) { - DCV_dealloc(&self->vec); + DCV_dealloc(&(self->vec)); } static @@ -305,7 +333,7 @@ static PyTypeObject DeltaChunkListType = { 0, /* tp_dictoffset */ (initproc)DCL_init, /* tp_init */ 0, /* tp_alloc */ - (newfunc)DCL_new, /* tp_new */ + 0, /* tp_new */ }; diff --git a/stream.py b/stream.py index 169104625..6d7479d1c 100644 --- a/stream.py +++ b/stream.py @@ -339,6 +339,9 @@ def _set_cache_(self, attr): # the final delta data stream. dcl = connect_deltas(self._dstreams) assert dcl is not None + print "got dcl" + del(dcl) + print "dealloc worked" # call len directly, as the (optional) c version doesn't implement the sequence # protocol From 445b71637576649fd6f1f3c287f50eec24d4fbf4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 12:43:57 +0200 Subject: [PATCH 0080/1392] Wow, this was a lesson. My full hatred goes to python, and C, and everything ... cool if you control everything, but not cool if an Object_New call doesn't do anything for you - creating a new instance of an own type in python doesn't appear to be that easy after all, at least not if you want your initializers/new procs to be called --- _fun.c | 60 +++++++++++++++++++++++++++++++------------------------ stream.py | 3 --- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/_fun.c b/_fun.c index 47f5d8ecf..3a1bd0355 100644 --- a/_fun.c +++ b/_fun.c @@ -192,13 +192,12 @@ DeltaChunk* DCV_end(DeltaChunkVector* vec) void DCV_dealloc(DeltaChunkVector* vec) { if (vec->mem){ - if (vec->size){ - const DeltaChunk* end = DCV_end(vec); - DeltaChunk* i; - for(i = &vec->mem[0]; i < end; i++){ - DC_destroy(i); - } + const DeltaChunk* end = DCV_end(vec); + DeltaChunk* i; + for(i = vec->mem; i < end; i++){ + DC_destroy(i); } + PyMem_Free(vec->mem); vec->size = 0; vec->reserved_size = 0; @@ -207,7 +206,7 @@ void DCV_dealloc(DeltaChunkVector* vec) } // Append num-chunks to the end of the list, possibly reallocating existing ones -// Return a pointer to the first of the added items. They are not yet initialized +// Return a pointer to the first of the added items. They are already null initialized // If num-chunks == 0, it returns the end pointer of the allocated memory static inline DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) @@ -220,6 +219,11 @@ DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) Py_FatalError("Could not allocate memory for append operation"); Py_ssize_t old_size = vec->size; vec->size += num_chunks; + + for(;old_size < vec->size; ++old_size){ + DC_init(DCV_get(vec, old_size), 0, 0, 0, NULL); + } + return &vec->mem[old_size]; } @@ -235,21 +239,15 @@ typedef struct { static -int DCL_init(DeltaChunkList *self, PyObject *args, PyObject *kwds) +int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) { - if(PySequence_Size(args) > 1){ - PyErr_SetString(PyExc_ValueError, "Zero or one arguments are allowed, providing the initial size of the queue in DeltaChunks"); - return 0; + if(args && PySequence_Size(args) > 0){ + PyErr_SetString(PyExc_ValueError, "Too many arguments"); + return -1; } - assert(self->vec.mem == NULL); - - ull init_size = 0; - PyArg_ParseTuple(args, "K", &init_size); - if (init_size == 0){ - init_size = 12500; - } - return DCV_init(&self->vec, init_size); + DCV_init(&self->vec, 0); + return 0; } static @@ -285,8 +283,6 @@ PyObject* DCL_apply(PyObject* self, PyObject* args) Py_RETURN_NONE; } - - static PyMethodDef DCL_methods[] = { {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, @@ -331,12 +327,24 @@ static PyTypeObject DeltaChunkListType = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - (initproc)DCL_init, /* tp_init */ + (initproc)DCL_init, /* tp_init */ 0, /* tp_alloc */ - 0, /* tp_new */ + 0, /* tp_new */ }; +// Makes a new copy of the DeltaChunkList - you have to do everything yourselve +// in C ... want C++ !! +DeltaChunkList* DCL_new_instance(void) +{ + DeltaChunkList* dcl = (DeltaChunkList*) PyType_GenericNew(&DeltaChunkListType, 0, 0); + assert(dcl); + + DCL_init(dcl, 0, 0); + assert(dcl->vec.size == 0); + return dcl; +} + static inline ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* out_bytes_read){ ull size = 0; @@ -351,7 +359,7 @@ ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* o }// END while in range *out_bytes_read = i+offset; - assert((*out_bytes_read * 8) - (*out_bytes_read - 1) <= sizeof(ull)); + assert((*out_bytes_read * 8) - (*out_bytes_read - 1) <= sizeof(ull) * 8); return size; } @@ -373,7 +381,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DeltaChunkList* tdcl = 0; DeltaChunkList* dcl = 0; - dcl = tdcl = PyObject_New(DeltaChunkList, &DeltaChunkListType); + dcl = tdcl = DCL_new_instance(); + assert(dcl != NULL); if (!dcl){ PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); return NULL; @@ -478,7 +487,6 @@ PyMODINIT_FUNC init_fun(void) { PyObject *m; - DeltaChunkListType.tp_new = PyType_GenericNew; if (PyType_Ready(&DeltaChunkListType) < 0) return; diff --git a/stream.py b/stream.py index 6d7479d1c..169104625 100644 --- a/stream.py +++ b/stream.py @@ -339,9 +339,6 @@ def _set_cache_(self, attr): # the final delta data stream. dcl = connect_deltas(self._dstreams) assert dcl is not None - print "got dcl" - del(dcl) - print "dealloc worked" # call len directly, as the (optional) c version doesn't implement the sequence # protocol From 9ba93c0deb4f46b524e28c8103b35f408a936e04 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 14:58:38 +0200 Subject: [PATCH 0081/1392] Apparently, the most serious memory bugs are fixed for now, lets get back to the actual thing --- _fun.c | 73 +++++++++++++++++++++++++++++++++++++++++++------------ stream.py | 1 - 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/_fun.c b/_fun.c index 3a1bd0355..7089cfe3d 100644 --- a/_fun.c +++ b/_fun.c @@ -2,6 +2,7 @@ #include #include #include +#include static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -87,7 +88,6 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) typedef unsigned long long ull; typedef unsigned int uint; - // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -142,13 +142,17 @@ int DCV_grow(DeltaChunkVector* vec, uint num_dc) } if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(grow_by_chunks * sizeof(vec->mem)); + vec->mem = PyMem_Malloc(grow_by_chunks * sizeof(DeltaChunk)); } else { - vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(vec->mem)); + vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)); } assert(vec->mem != NULL); vec->reserved_size = vec->reserved_size + grow_by_chunks; +#ifdef DEBUG + fprintf(stderr, "Allocated %i bytes at %p, to hold up to %i chunks\n", (int)((vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)), vec->mem, (int)(vec->reserved_size + grow_by_chunks)); +#endif + return vec->mem != NULL; } @@ -192,7 +196,11 @@ DeltaChunk* DCV_end(DeltaChunkVector* vec) void DCV_dealloc(DeltaChunkVector* vec) { if (vec->mem){ - const DeltaChunk* end = DCV_end(vec); +#ifdef DEBUG + fprintf(stderr, "Freeing %p\n", (void*)vec->mem); +#endif + + const DeltaChunk* end = &vec->mem[vec->size]; DeltaChunk* i; for(i = vec->mem; i < end; i++){ DC_destroy(i); @@ -342,6 +350,7 @@ DeltaChunkList* DCL_new_instance(void) DCL_init(dcl, 0, 0); assert(dcl->vec.size == 0); + assert(dcl->vec.mem == NULL); return dcl; } @@ -377,16 +386,13 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) stream_iter = dstreams; } - DeltaChunkList* bdcl = 0; - DeltaChunkList* tdcl = 0; - DeltaChunkList* dcl = 0; + DeltaChunkVector bdcv; + DeltaChunkVector tdcv; + DeltaChunkVector dcv; + DCV_init(&bdcv, 0); + DCV_init(&dcv, 0); + DCV_init(&tdcv, 0); - dcl = tdcl = DCL_new_instance(); - assert(dcl != NULL); - if (!dcl){ - PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); - return NULL; - } unsigned int dsi; PyObject* ds; @@ -408,6 +414,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_ssize_t ofs = 0; const ull base_size = msb_size(data, dlen, 0, &ofs); const ull target_size = msb_size(data, dlen, ofs, &ofs); + + // estimate number of ops - assume one third adds, half two byte (size+offset) copies + const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); + DCV_grow(&dcv, approx_num_cmds); // parse command stream const char* dend = data + dlen; @@ -431,7 +441,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const unsigned long rbound = cp_off + cp_size; if (rbound < cp_size || rbound > base_size){ - goto loop_end; + break; } // TODO: Add node @@ -446,8 +456,19 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) goto loop_end; } }// END handle command opcodes - assert(tbw == target_size); + + // swap the vector + // Skip the first vector, as it is also used as top chunk vector + if (bdcv.mem != tdcv.mem){ + DCV_dealloc(&bdcv); + } + bdcv = dcv; + if (dsi == 0){ + tdcv = dcv; + } + DCV_init(&dcv, 0); + loop_end: // perform cleanup @@ -467,11 +488,31 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(stream_iter); } + DCV_dealloc(&bdcv); + if (dsi > 1){ + // otherwise dcv equals tcl + DCV_dealloc(&dcv); + } + + // Return the actual python object - its just a container + DeltaChunkList* dcl = DCL_new_instance(); + if (!dcl){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); + // Otherwise tdcv would be deallocated by the chunk list + DCV_dealloc(&tdcv); + error = 1; + } else { + // Plain copy, don't deallocate + dcl->vec = tdcv; + } + if (error){ + // Will dealloc tdcv + Py_XDECREF(dcl); return NULL; } - return (PyObject*)tdcl; + return (PyObject*)dcl; } static PyMethodDef py_fun[] = { diff --git a/stream.py b/stream.py index 169104625..af4591f85 100644 --- a/stream.py +++ b/stream.py @@ -338,7 +338,6 @@ def _set_cache_(self, attr): # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. dcl = connect_deltas(self._dstreams) - assert dcl is not None # call len directly, as the (optional) c version doesn't implement the sequence # protocol From 489f763308d4982a5220a648b92ecb2cc82f4ae4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 16:04:55 +0200 Subject: [PATCH 0082/1392] Now adding chunks to the vectors, next up is to implement the actual chunk merging --- _fun.c | 104 ++++++++++++++++++++++++++++++++++++++------------------- fun.py | 1 + 2 files changed, 71 insertions(+), 34 deletions(-) diff --git a/_fun.c b/_fun.c index 7089cfe3d..2ad8d641d 100644 --- a/_fun.c +++ b/_fun.c @@ -3,6 +3,7 @@ #include #include #include +#include static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -87,6 +88,7 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) typedef unsigned long long ull; typedef unsigned int uint; +typedef unsigned char uchar; // DELTA CHUNK //////////////// @@ -95,21 +97,38 @@ typedef struct { ull to; ull ts; ull so; - PyObject* data; + uchar* data; } DeltaChunk; -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, PyObject* data) +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) { dc->to = to; dc->ts = ts; dc->so = so; - Py_XINCREF(data); - dc->data = data; + dc->data = NULL; } void DC_destroy(DeltaChunk* dc) { - Py_XDECREF(dc->data); + if (dc->data){ + PyMem_Free((void*)dc->data); + } +} + +// Store a copy of data in our instance +void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen) +{ + if (dc->data){ + PyMem_Free((void*)dc->data); + } + + if (data == 0){ + dc->data = NULL; + return; + } + + dc->data = (uchar*)PyMem_Malloc(dlen); + memcpy(dc->data, data, dlen); } ull DC_rbound(DeltaChunk* dc) @@ -146,7 +165,11 @@ int DCV_grow(DeltaChunkVector* vec, uint num_dc) } else { vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)); } - assert(vec->mem != NULL); + + if (vec->mem == NULL){ + Py_FatalError("Could not allocate memory for append operation"); + } + vec->reserved_size = vec->reserved_size + grow_by_chunks; #ifdef DEBUG @@ -220,21 +243,33 @@ static inline DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) { if (vec->size + num_chunks > vec->reserved_size){ - if (!DCV_grow(vec, (vec->size + num_chunks) - vec->reserved_size)){ - Py_FatalError("Could not allocate memory for append operation"); - } + DCV_grow(vec, (vec->size + num_chunks) - vec->reserved_size); } Py_FatalError("Could not allocate memory for append operation"); Py_ssize_t old_size = vec->size; vec->size += num_chunks; for(;old_size < vec->size; ++old_size){ - DC_init(DCV_get(vec, old_size), 0, 0, 0, NULL); + DC_init(DCV_get(vec, old_size), 0, 0, 0); } return &vec->mem[old_size]; } +// Append one chunk to the end of the list, and return a pointer to it +// It will have been initialized. +static inline +DeltaChunk* DCV_append(DeltaChunkVector* vec) +{ + if (vec->size + 1 > vec->reserved_size){ + DCV_grow(vec, 1); + } + + DeltaChunk* next = vec->mem + vec->size; + vec->size += 1; + return next; +} + // DELTA CHUNK LIST (PYTHON) ///////////////////////////// @@ -354,21 +389,18 @@ DeltaChunkList* DCL_new_instance(void) return dcl; } -static inline -ull msb_size(const char* data, Py_ssize_t dlen, Py_ssize_t offset, Py_ssize_t* out_bytes_read){ - ull size = 0; - Py_ssize_t i = 0; - const char* dend = data + dlen; - for (data = data + offset; data < dend; data+=1, i+=1){ - char c = *data; - size |= (c & 0x7f) << i*7; - if (!(c & 0x80)){ - break; - } - }// END while in range - - *out_bytes_read = i+offset; - assert((*out_bytes_read * 8) - (*out_bytes_read - 1) <= sizeof(ull) * 8); +inline +ull msb_size(const uchar** datap, const uchar* top) +{ + const uchar *data = *datap; + ull cmd, size = 0; + uint i = 0; + do { + cmd = *data++; + size |= (cmd & 0x7f) << i; + i += 7; + } while (cmd & 0x80 && data < top); + *datap = data; return size; } @@ -406,25 +438,25 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) goto loop_end; } - const char* data; + const uchar* data; Py_ssize_t dlen; PyObject_AsReadBuffer(db, (const void**)&data, &dlen); + const uchar* dend = data + dlen; // read header - Py_ssize_t ofs = 0; - const ull base_size = msb_size(data, dlen, 0, &ofs); - const ull target_size = msb_size(data, dlen, ofs, &ofs); + const ull base_size = msb_size(&data, dend); + const ull target_size = msb_size(&data, dend); // estimate number of ops - assume one third adds, half two byte (size+offset) copies const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); DCV_grow(&dcv, approx_num_cmds); // parse command stream - const char* dend = data + dlen; ull tbw = 0; // Amount of target bytes written - for (data = data + ofs; data < dend; ++data) + assert(data < dend); + while (data < dend) { - const char cmd = *data; + const char cmd = *data++; if (cmd & 0x80) { @@ -444,12 +476,16 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) break; } - // TODO: Add node + DC_init(DCV_append(&dcv), tbw, cp_size, cp_off); tbw += cp_size; } else if (cmd) { - // TODO: Add node + // TODO: Compress nodes by parsing them in advance + DeltaChunk* dc = DCV_append(&dcv); + DC_init(dc, tbw, cmd, 0); + DC_set_data(dc, data, cmd); tbw += cmd; + data += cmd; } else { error = 1; PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); diff --git a/fun.py b/fun.py index a4da30903..e6262b4bc 100644 --- a/fun.py +++ b/fun.py @@ -703,6 +703,7 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: + # raise ImportError; # DEBUG from _fun import connect_deltas except ImportError: pass From a93363cffb225520869d737de1081c1ff77ed108 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 17:45:42 +0200 Subject: [PATCH 0083/1392] prepared the slicing, as well as a few accompanying methods. There is still quite a lot functionality missing --- _fun.c | 194 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 166 insertions(+), 28 deletions(-) diff --git a/_fun.c b/_fun.c index 2ad8d641d..1186c6d11 100644 --- a/_fun.c +++ b/_fun.c @@ -89,6 +89,7 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) typedef unsigned long long ull; typedef unsigned int uint; typedef unsigned char uchar; +typedef uchar bool; // DELTA CHUNK //////////////// @@ -97,45 +98,96 @@ typedef struct { ull to; ull ts; ull so; - uchar* data; + const uchar* data; + bool data_shared; } DeltaChunk; +inline void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) { dc->to = to; dc->ts = ts; dc->so = so; dc->data = NULL; + dc->data_shared = 0; } -void DC_destroy(DeltaChunk* dc) +inline +void DC_deallocate_data(DeltaChunk* dc) { - if (dc->data){ + if (!dc->data_shared && dc->data){ PyMem_Free((void*)dc->data); } + dc->data = NULL; } -// Store a copy of data in our instance -void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen) +inline +void DC_destroy(DeltaChunk* dc) { - if (dc->data){ - PyMem_Free((void*)dc->data); - } + DC_deallocate_data(dc); +} + +// Store a copy of data in our instance. If shared is 1, the data will be shared, +// hence it will only be stored, but the memory will not be touched, or copied. +inline +void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared) +{ + DC_deallocate_data(dc); if (data == 0){ dc->data = NULL; + dc->data_shared = 0; return; } - dc->data = (uchar*)PyMem_Malloc(dlen); - memcpy(dc->data, data, dlen); + dc->data_shared = shared; + if (shared){ + dc->data = data; + } else { + dc->data = (uchar*)PyMem_Malloc(dlen); + memcpy((void*)dc->data, (void*)data, dlen); + } + } +inline ull DC_rbound(DeltaChunk* dc) { return dc->to + dc->ts; } +// Copy all data from src to dest, the data pointer will be copied too +inline +void DC_copy_to(DeltaChunk* src, DeltaChunk* dest) +{ + dest->to = src->to; + dest->ts = src->ts; + dest->so = src->so; + dest->data_shared = 0; + + DC_set_data(dest, src->data, src->ts, 0); +} + +// Copy all data with the given offset and size. The source offset, as well +// as the data will be truncated accordingly +inline +void DC_offset_copy_to(DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) +{ + assert(size <= src->ts); + assert(src->to + ofs + size <= DC_rbound(src)); + + dest->to = src->to + ofs; + dest->ts = size; + dest->so = src->so + ofs; + + if (src->data){ + DC_set_data(dest, src->data + ofs, size, 0); + } else { + dest->data = NULL; + dest->data_shared = 0; + } +} + // DELTA CHUNK VECTOR ///////////////////// @@ -152,7 +204,7 @@ This may trigger a realloc, but will do nothing if the reserved size is already large enough. Return 1 on success, 0 on failure */ -static +inline int DCV_grow(DeltaChunkVector* vec, uint num_dc) { const uint grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; @@ -188,35 +240,48 @@ int DCV_init(DeltaChunkVector* vec, ull initial_size) return DCV_grow(vec, initial_size); } -static inline +inline ull DCV_len(DeltaChunkVector* vec) { return vec->size; } +inline +ull DCV_lbound(DeltaChunkVector* vec) +{ + assert(vec->size && vec->mem); + return vec->mem->to; +} + // Return item at index -static inline +inline DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) { assert(i < vec->size && vec->mem); return &vec->mem[i]; } -static inline +inline +ull DCV_rbound(DeltaChunkVector* vec) +{ + return DC_rbound(DCV_get(vec, vec->size-1)); +} + +inline int DCV_empty(DeltaChunkVector* vec) { return vec->size == 0; } // Return end pointer of the vector -static inline +inline DeltaChunk* DCV_end(DeltaChunkVector* vec) { assert(!DCV_empty(vec)); return &vec->mem[vec->size]; } -void DCV_dealloc(DeltaChunkVector* vec) +void DCV_destroy(DeltaChunkVector* vec) { if (vec->mem){ #ifdef DEBUG @@ -236,6 +301,14 @@ void DCV_dealloc(DeltaChunkVector* vec) } } +// Reset this vector so that its existing memory can be filled again. +// Memory will be kept, but not cleaned up +inline +void DCV_forget_members(DeltaChunkVector* vec) +{ + vec->size = 0; +} + // Append num-chunks to the end of the list, possibly reallocating existing ones // Return a pointer to the first of the added items. They are already null initialized // If num-chunks == 0, it returns the end pointer of the allocated memory @@ -249,15 +322,17 @@ DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) Py_ssize_t old_size = vec->size; vec->size += num_chunks; +#ifdef DEBUG for(;old_size < vec->size; ++old_size){ DC_init(DCV_get(vec, old_size), 0, 0, 0); } +#endif return &vec->mem[old_size]; } // Append one chunk to the end of the list, and return a pointer to it -// It will have been initialized. +// It will not have been initialized ! static inline DeltaChunk* DCV_append(DeltaChunkVector* vec) { @@ -270,6 +345,59 @@ DeltaChunk* DCV_append(DeltaChunkVector* vec) return next; } +// Write a slice as defined by its absolute offset in bytes and its size into the given +// destination. The individual chunks written will be a deep copy of the source +// data chunks +// TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort +// of append-only memory pool would improve performance +inline +void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) +{ + +} + + +// Take slices of bdcv into the corresponding area of the tdcv, which is the topmost +// delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the +// caller +static +void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) +{ + DeltaChunk* dc = tdcv->mem; + DeltaChunk* end = tdcv->mem + tdcv->size; + assert(dc); + + for (;dc < end; dc++) + { + // Data chunks don't need processing + if (dc->data){ + continue; + } + + // Copy Chunk Handling + DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); + // assert(tmpl->size); + + // move target bounds + DeltaChunk* cdc = tmpl->mem; + DeltaChunk* cdcend = tmpl->mem + tmpl->size; + const ull ofs = dc->to - dc->so; + for(;cdc < cdcend; cdc++){ + cdc->to += ofs; + } + + // insert slice into our list, replacing our current chunk + if (tmpl->size == 1){ + *dc = *DCV_get(tmpl, 0); + } else { + + } + + // make sure the members will not be deallocated by the list + DCV_forget_members(tmpl); + } +} + // DELTA CHUNK LIST (PYTHON) ///////////////////////////// @@ -296,7 +424,7 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) static void DCL_dealloc(DeltaChunkList* self) { - DCV_dealloc(&(self->vec)); + DCV_destroy(&(self->vec)); } static @@ -310,7 +438,7 @@ ull DCL_rbound(DeltaChunkList* self) { if (DCV_empty(&self->vec)) return 0; - return DC_rbound(DCV_get(&self->vec, self->vec.size - 1)); + return DCV_rbound(&self->vec); } static @@ -421,10 +549,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DeltaChunkVector bdcv; DeltaChunkVector tdcv; DeltaChunkVector dcv; + DeltaChunkVector tmpl; DCV_init(&bdcv, 0); DCV_init(&dcv, 0); DCV_init(&tdcv, 0); - + DCV_init(&tmpl, 200); unsigned int dsi; PyObject* ds; @@ -453,6 +582,9 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // parse command stream ull tbw = 0; // Amount of target bytes written + bool shared_data = dsi != 0; + bool is_first_run = dsi == 0; + assert(data < dend); while (data < dend) { @@ -481,9 +613,12 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } else if (cmd) { // TODO: Compress nodes by parsing them in advance + // NOTE: Compression only necessary for all other deltas, not + // for the first one, as we will share the data. It really depends + // What's faster DeltaChunk* dc = DCV_append(&dcv); DC_init(dc, tbw, cmd, 0); - DC_set_data(dc, data, cmd); + DC_set_data(dc, data, cmd, shared_data); tbw += cmd; data += cmd; } else { @@ -493,18 +628,20 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } }// END handle command opcodes assert(tbw == target_size); - + + if (!is_first_run){ + DCV_connect_with_base(&tdcv, &dcv, &tmpl); + } // swap the vector // Skip the first vector, as it is also used as top chunk vector if (bdcv.mem != tdcv.mem){ - DCV_dealloc(&bdcv); + DCV_destroy(&bdcv); } bdcv = dcv; - if (dsi == 0){ + if (is_first_run){ tdcv = dcv; } DCV_init(&dcv, 0); - loop_end: // perform cleanup @@ -524,10 +661,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(stream_iter); } - DCV_dealloc(&bdcv); + DCV_destroy(&tmpl); + DCV_destroy(&bdcv); if (dsi > 1){ // otherwise dcv equals tcl - DCV_dealloc(&dcv); + DCV_destroy(&dcv); } // Return the actual python object - its just a container @@ -535,7 +673,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (!dcl){ PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); // Otherwise tdcv would be deallocated by the chunk list - DCV_dealloc(&tdcv); + DCV_destroy(&tdcv); error = 1; } else { // Plain copy, don't deallocate From 166e538f9aab8db7ab30b6e8b3be407200a7e3c1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 18:17:19 +0200 Subject: [PATCH 0084/1392] Enhanced memory handling within the delta-stream parsing method. Removed the base delta chunk vector, which was a reminder of old (python) times which are long gone --- _fun.c | 88 +++++++++++++++++++++++++++++++++++++++------------------- fun.py | 4 +-- 2 files changed, 60 insertions(+), 32 deletions(-) diff --git a/_fun.c b/_fun.c index 1186c6d11..142795e6c 100644 --- a/_fun.c +++ b/_fun.c @@ -198,46 +198,62 @@ typedef struct { Py_ssize_t reserved_size; // Reserve in DeltaChunks } DeltaChunkVector; -/* -Grow the delta chunk list by the given amount of bytes. -This may trigger a realloc, but will do nothing if the reserved size is already -large enough. -Return 1 on success, 0 on failure -*/ + + +// Reserve enough memory to hold the given amount of delta chunks +// Return 1 on success inline -int DCV_grow(DeltaChunkVector* vec, uint num_dc) +int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) { - const uint grow_by_chunks = (vec->size + num_dc) - vec->reserved_size; - if (grow_by_chunks <= 0){ + if (num_dc <= vec->reserved_size){ return 1; } +#ifdef DEBUG + bool was_null = vec->mem == NULL; +#endif + if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(grow_by_chunks * sizeof(DeltaChunk)); + vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaChunk)); } else { - vec->mem = PyMem_Realloc(vec->mem, (vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)); + vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaChunk)); } if (vec->mem == NULL){ Py_FatalError("Could not allocate memory for append operation"); } - vec->reserved_size = vec->reserved_size + grow_by_chunks; + vec->reserved_size = num_dc; #ifdef DEBUG - fprintf(stderr, "Allocated %i bytes at %p, to hold up to %i chunks\n", (int)((vec->reserved_size + grow_by_chunks) * sizeof(DeltaChunk)), vec->mem, (int)(vec->reserved_size + grow_by_chunks)); + const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; + if (!was_null) + format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; + fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaChunk)), vec->mem, (int)vec->reserved_size); #endif return vec->mem != NULL; } +/* +Grow the delta chunk list by the given amount of bytes. +This may trigger a realloc, but will do nothing if the reserved size is already +large enough. +Return 1 on success, 0 on failure +*/ +inline +int DCV_grow_by(DeltaChunkVector* vec, uint num_dc) +{ + return DCV_reserve_memory(vec, vec->reserved_size + num_dc); +} + int DCV_init(DeltaChunkVector* vec, ull initial_size) { vec->mem = NULL; vec->size = 0; vec->reserved_size = 0; - return DCV_grow(vec, initial_size); + return DCV_grow_by(vec, initial_size); } inline @@ -309,6 +325,24 @@ void DCV_forget_members(DeltaChunkVector* vec) vec->size = 0; } +// Reset the vector so that its size will be zero, and its members will +// have been deallocated properly. +// It will keep its memory though, and hence can be filled again +inline +void DCV_reset(DeltaChunkVector* vec) +{ + if (vec->size == 0) + return; + + DeltaChunk* dc = vec->mem; + DeltaChunk* dcend = DCV_end(vec); + for(;dc < dcend; dc++){ + DC_destroy(dc); + } + + vec->size = 0; +} + // Append num-chunks to the end of the list, possibly reallocating existing ones // Return a pointer to the first of the added items. They are already null initialized // If num-chunks == 0, it returns the end pointer of the allocated memory @@ -316,7 +350,7 @@ static inline DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) { if (vec->size + num_chunks > vec->reserved_size){ - DCV_grow(vec, (vec->size + num_chunks) - vec->reserved_size); + DCV_grow_by(vec, (vec->size + num_chunks) - vec->reserved_size); } Py_FatalError("Could not allocate memory for append operation"); Py_ssize_t old_size = vec->size; @@ -337,7 +371,7 @@ static inline DeltaChunk* DCV_append(DeltaChunkVector* vec) { if (vec->size + 1 > vec->reserved_size){ - DCV_grow(vec, 1); + DCV_grow_by(vec, 1); } DeltaChunk* next = vec->mem + vec->size; @@ -546,12 +580,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) stream_iter = dstreams; } - DeltaChunkVector bdcv; - DeltaChunkVector tdcv; DeltaChunkVector dcv; + DeltaChunkVector tdcv; DeltaChunkVector tmpl; - DCV_init(&bdcv, 0); - DCV_init(&dcv, 0); + DCV_init(&dcv, 100); // should be enough to keep the average text file DCV_init(&tdcv, 0); DCV_init(&tmpl, 200); @@ -578,7 +610,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // estimate number of ops - assume one third adds, half two byte (size+offset) copies const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); - DCV_grow(&dcv, approx_num_cmds); + DCV_reserve_memory(&dcv, approx_num_cmds); // parse command stream ull tbw = 0; // Amount of target bytes written @@ -632,16 +664,15 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (!is_first_run){ DCV_connect_with_base(&tdcv, &dcv, &tmpl); } - // swap the vector - // Skip the first vector, as it is also used as top chunk vector - if (bdcv.mem != tdcv.mem){ - DCV_destroy(&bdcv); - } - bdcv = dcv; + if (is_first_run){ tdcv = dcv; + // wipe out dcv without destroying the members, get its own memory + DCV_init(&dcv, tdcv.size); + } else { + // destroy members, but keep memory + DCV_reset(&dcv); } - DCV_init(&dcv, 0); loop_end: // perform cleanup @@ -662,7 +693,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } DCV_destroy(&tmpl); - DCV_destroy(&bdcv); if (dsi > 1){ // otherwise dcv equals tcl DCV_destroy(&dcv); diff --git a/fun.py b/fun.py index e6262b4bc..13a3c627f 100644 --- a/fun.py +++ b/fun.py @@ -545,7 +545,6 @@ def connect_deltas(dstreams): :param dstreams: iterable of delta stream objects, the delta to be applied last comes first, then all its ancestors in order :return: DeltaChunkList, containing all operations to apply""" - bdcl = None # data chunk list for initial base tdcl = None # topmost dcl dcl = tdcl = TopdownDeltaChunkList() @@ -611,13 +610,12 @@ def connect_deltas(dstreams): dcl.compress() # merge the lists ! - if bdcl is not None: + if dsi > 0: if not tdcl.connect_with_next_base(dcl): break # END handle merge # prepare next base - bdcl = dcl dcl = DeltaChunkList() # END for each delta stream From 60f7768ed00ad666317c1877abe52906d6014d50 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 19:46:24 +0200 Subject: [PATCH 0085/1392] Implemented everything about the merging of the bases into the topmost delta list. Its not yet working, but at least its not crashing --- _fun.c | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 101 insertions(+), 9 deletions(-) diff --git a/_fun.c b/_fun.c index 142795e6c..eb4b6b865 100644 --- a/_fun.c +++ b/_fun.c @@ -91,6 +91,9 @@ typedef unsigned int uint; typedef unsigned char uchar; typedef uchar bool; +// Constants +const ull gDVC_grow_by = 50; + // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -277,10 +280,17 @@ DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) return &vec->mem[i]; } +// Return last item +inline +DeltaChunk* DCV_last(DeltaChunkVector* vec) +{ + return DCV_get(vec, vec->size-1); +} + inline ull DCV_rbound(DeltaChunkVector* vec) { - return DC_rbound(DCV_get(vec, vec->size-1)); + return DC_rbound(DCV_last(vec)); } inline @@ -371,7 +381,7 @@ static inline DeltaChunk* DCV_append(DeltaChunkVector* vec) { if (vec->size + 1 > vec->reserved_size){ - DCV_grow_by(vec, 1); + DCV_grow_by(vec, gDVC_grow_by); } DeltaChunk* next = vec->mem + vec->size; @@ -379,6 +389,33 @@ DeltaChunk* DCV_append(DeltaChunkVector* vec) return next; } +// Return delta chunk being closest to the given absolute offset +inline +DeltaChunk* DCV_closest_chunk(DeltaChunkVector* vec, ull ofs) +{ + assert(vec->mem); + + ull lo = 0; + ull hi = vec->size; + ull mid; + DeltaChunk* dc; + + while (lo < hi) + { + mid = (lo + hi) / 2; + dc = vec->mem + mid; + if (dc->to > ofs){ + hi = mid; + } else if ((DC_rbound(dc) > ofs) | (dc->to == ofs)) { + return dc; + } else { + lo = mid + 1; + } + } + + return DCV_last(vec); +} + // Write a slice as defined by its absolute offset in bytes and its size into the given // destination. The individual chunks written will be a deep copy of the source // data chunks @@ -387,14 +424,67 @@ DeltaChunk* DCV_append(DeltaChunkVector* vec) inline void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) { + assert(DCV_lbound(src) <= ofs); + assert(DCV_rbound(src) <= ofs + size); + DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + + // partial overlap + if (cdc->to != ofs) { + DeltaChunk* destc = DCV_append(dest); + const ull relofs = ofs - cdc->to; + DC_offset_copy_to(cdc, destc, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); + cdc += 1; + size -= destc->ts; + + if (size == 0){ + return; + } + } + + DeltaChunk* vecend = DCV_end(src); + for( ;(cdc < vecend) && size; ++cdc) + { + if (cdc->ts < size) { + DC_copy_to(cdc, DCV_append(dest)); + size -= cdc->ts; + } else { + DC_offset_copy_to(cdc, DCV_append(dest), 0, size); + size = 0; + break; + } + } + + assert(size == 0); } +// Insert all chunks in 'from' to 'to', starting at the delta chunk named 'at' which +// originates in to +// 'at' will be replaced by the items to insert ( special purpose ) +// 'at' will be properly destroyed, but all items will just be copied bytewise +// using memcpy. Hence from must just forget about them ! +inline +void DCV_replace_one_by_many(DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) +{ + assert(from->size > 1); + + DCV_reserve_memory(to, to->size + from->size - 1); // -1 because we replace at + DC_destroy(at); + to->size -= 1 + from->size; + + // If we are somewhere in the middle, we have to make some space + if (DCV_last(to) != at) { + memmove((void*)at+from->size, (void*)(at+1), (size_t)(DCV_end(to) - (at+1))); + } + + // Finally copy all the items in + memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); +} + // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost // delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the // caller -static void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) { DeltaChunk* dc = tdcv->mem; @@ -413,18 +503,20 @@ void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, Delta // assert(tmpl->size); // move target bounds - DeltaChunk* cdc = tmpl->mem; - DeltaChunk* cdcend = tmpl->mem + tmpl->size; + DeltaChunk* tdc = tmpl->mem; + DeltaChunk* tdcend = tmpl->mem + tmpl->size; const ull ofs = dc->to - dc->so; - for(;cdc < cdcend; cdc++){ - cdc->to += ofs; + for(;tdc < tdcend; tdc++){ + tdc->to += ofs; } - // insert slice into our list, replacing our current chunk + // insert slice into our list if (tmpl->size == 1){ + // Its not data, so destroy is not really required, anyhow ... + DC_destroy(dc); *dc = *DCV_get(tmpl, 0); } else { - + DCV_replace_one_by_many(tmpl, tdcv, dc); } // make sure the members will not be deallocated by the list From 60b4f37a8b17c8f89b1b0ba7a0268f375469033d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 22:06:27 +0200 Subject: [PATCH 0086/1392] Improved performance of python implementation by 10 percent, just by removing function calls and object creations --- fun.py | 82 ++++++++++++---------------------------------------------- 1 file changed, 17 insertions(+), 65 deletions(-) diff --git a/fun.py b/fun.py index 13a3c627f..038b4c761 100644 --- a/fun.py +++ b/fun.py @@ -55,9 +55,6 @@ def _set_delta_rbound(d, size): :param size: size relative to our target offset, may not be 0, must be smaller or equal to our size :return: d""" - if d.ts == size: - return - d.ts = size # NOTE: data is truncated automatically when applying the delta @@ -154,76 +151,30 @@ def _closest_index(dcl, absofs): # END for each delta absofs return len(dcl)-1 -def delta_list_apply(dcl, bbuf, write, lbound_offset=0, size=0): +def delta_list_apply(dcl, bbuf, write): """Apply the chain's changes and write the final result using the passed write function. :param bbuf: base buffer containing the base of all deltas contained in this list. It will only be used if the chunk in question does not have a base chain. - :param lbound_offset: offset at which to start applying the delta, relative to - our lbound - :param size: if larger than 0, only the given amount of bytes will be applied :param write: function taking a string of bytes to write to the output""" - slen = len(dcl) - if slen == 0: - return - # END early abort - absofs = dcl.lbound() + lbound_offset - if size == 0: - size = dcl.rbound() - absofs - # END initialize size - - if lbound_offset or absofs + size != dcl.rbound(): - cdi = _closest_index(dcl, absofs) - cd = dcl[cdi] - if cd.to != absofs: - tcd = delta_duplicate(cd) - _move_delta_lbound(tcd, absofs - cd.to) - _set_delta_rbound(tcd, min(tcd.ts, size)) - delta_chunk_apply(tcd, bbuf, write) - size -= tcd.ts - cdi += 1 - # END handle first chunk - - # here we have to either apply full chunks, or smaller ones, but - # we always start at the chunks target offset - while cdi < slen and size: - cd = dcl[cdi] - if cd.ts <= size: - delta_chunk_apply(cd, bbuf, write) - size -= cd.ts - else: - tcd = delta_duplicate(cd) - _set_delta_rbound(tcd, size) - delta_chunk_apply(tcd, bbuf, write) - size -= tcd.ts - break - # END handle bytes to apply - cdi += 1 - # END handle rest - else: - for dc in dcl: - delta_chunk_apply(dc, bbuf, write) - # END for each dc - # END handle application values + for dc in dcl: + delta_chunk_apply(dc, bbuf, write) + # END for each dc -def delta_list_slice(dcl, absofs, size): +def delta_list_slice(dcl, absofs, size, ndcl): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. - :return: list (copy) which represents the given chunk""" - dcllbound = dcl.lbound() - absofs = max(absofs, dcllbound) - size = min(dcl.rbound() - dcllbound, size) + :return: None""" cdi = _closest_index(dcl, absofs) # delta start index cd = dcl[cdi] slen = len(dcl) - ndcl = list() lappend = ndcl.append if cd.to != absofs: - tcd = delta_duplicate(cd) + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) _move_delta_lbound(tcd, absofs - cd.to) - _set_delta_rbound(tcd, min(tcd.ts, size)) + tcd.ts = min(tcd.ts, size) lappend(tcd) size -= tcd.ts cdi += 1 @@ -233,11 +184,11 @@ def delta_list_slice(dcl, absofs, size): # are we larger than the current block cd = dcl[cdi] if cd.ts <= size: - lappend(delta_duplicate(cd)) + lappend(DeltaChunk(cd.to, cd.ts, cd.so, cd.data)) size -= cd.ts else: - tcd = delta_duplicate(cd) - _set_delta_rbound(tcd, size) + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) + tcd.ts = size lappend(tcd) size -= tcd.ts break @@ -245,7 +196,6 @@ def delta_list_slice(dcl, absofs, size): cdi += 1 # END for each chunk - return ndcl class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks. @@ -272,10 +222,10 @@ def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - def apply(self, bbuf, write, lbound_offset=0, size=0): + def apply(self, bbuf, write): """Only used by public clients, internally we only use the global routines for performance""" - return delta_list_apply(self, bbuf, write, lbound_offset, size) + return delta_list_apply(self, bbuf, write) def compress(self): """Alter the list to reduce the amount of nodes. Currently we concatenate @@ -369,6 +319,7 @@ def connect_with_next_base(self, bdcl): nfc = 0 # number of frozen chunks dci = 0 # delta chunk index slen = len(self) # len of self + ccl = list() # temporary list while dci < slen: dc = self[dci] dci += 1 @@ -384,8 +335,9 @@ def connect_with_next_base(self, bdcl): # dont support efficient insertion ( just one at a time ), but for now # we live with it. Internally, its all just a 32/64bit pointer, and # the portions of moved memory should be smallish. Maybe we just rebuild - # ourselves in order to reduce the amount of insertions ... - ccl = delta_list_slice(bdcl, dc.so, dc.ts) + # ourselves in order to reduce the amount of insertions ... + del(ccl[:]) + delta_list_slice(bdcl, dc.so, dc.ts, ccl) # move the target bounds into place to match with our chunk ofs = dc.to - dc.so From a63ee1d034a6677bc3ab408d1a16593bbb6078ca Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 12 Oct 2010 23:55:45 +0200 Subject: [PATCH 0087/1392] Currently there is a weird memory bug, valgrind says it is writing one byte too much. Perhaps its because of the use of PyMem --- _fun.c | 115 ++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 82 insertions(+), 33 deletions(-) diff --git a/_fun.c b/_fun.c index eb4b6b865..fc4ee1edc 100644 --- a/_fun.c +++ b/_fun.c @@ -94,6 +94,12 @@ typedef uchar bool; // Constants const ull gDVC_grow_by = 50; +#ifdef DEBUG +#define DBG_check(vec) DCV_dbg_check_integrity(vec) +#else +#define DBG_check(vec) +#endif + // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -154,19 +160,20 @@ void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared } inline -ull DC_rbound(DeltaChunk* dc) +ull DC_rbound(const DeltaChunk* dc) { return dc->to + dc->ts; } // Copy all data from src to dest, the data pointer will be copied too inline -void DC_copy_to(DeltaChunk* src, DeltaChunk* dest) +void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) { dest->to = src->to; dest->ts = src->ts; dest->so = src->so; dest->data_shared = 0; + dest->data = NULL; DC_set_data(dest, src->data, src->ts, 0); } @@ -174,7 +181,7 @@ void DC_copy_to(DeltaChunk* src, DeltaChunk* dest) // Copy all data with the given offset and size. The source offset, as well // as the data will be truncated accordingly inline -void DC_offset_copy_to(DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) +void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) { assert(size <= src->ts); assert(src->to + ofs + size <= DC_rbound(src)); @@ -182,6 +189,7 @@ void DC_offset_copy_to(DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) dest->to = src->to + ofs; dest->ts = size; dest->so = src->so + ofs; + dest->data = NULL; if (src->data){ DC_set_data(dest, src->data + ofs, size, 0); @@ -260,13 +268,13 @@ int DCV_init(DeltaChunkVector* vec, ull initial_size) } inline -ull DCV_len(DeltaChunkVector* vec) +ull DCV_len(const DeltaChunkVector* vec) { return vec->size; } inline -ull DCV_lbound(DeltaChunkVector* vec) +ull DCV_lbound(const DeltaChunkVector* vec) { assert(vec->size && vec->mem); return vec->mem->to; @@ -274,7 +282,7 @@ ull DCV_lbound(DeltaChunkVector* vec) // Return item at index inline -DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) +DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) { assert(i < vec->size && vec->mem); return &vec->mem[i]; @@ -282,29 +290,29 @@ DeltaChunk* DCV_get(DeltaChunkVector* vec, Py_ssize_t i) // Return last item inline -DeltaChunk* DCV_last(DeltaChunkVector* vec) +DeltaChunk* DCV_last(const DeltaChunkVector* vec) { return DCV_get(vec, vec->size-1); } inline -ull DCV_rbound(DeltaChunkVector* vec) +ull DCV_rbound(const DeltaChunkVector* vec) { return DC_rbound(DCV_last(vec)); } inline -int DCV_empty(DeltaChunkVector* vec) +int DCV_empty(const DeltaChunkVector* vec) { return vec->size == 0; } // Return end pointer of the vector inline -DeltaChunk* DCV_end(DeltaChunkVector* vec) +const DeltaChunk* DCV_end(const DeltaChunkVector* vec) { assert(!DCV_empty(vec)); - return &vec->mem[vec->size]; + return vec->mem + vec->size; } void DCV_destroy(DeltaChunkVector* vec) @@ -345,7 +353,7 @@ void DCV_reset(DeltaChunkVector* vec) return; DeltaChunk* dc = vec->mem; - DeltaChunk* dcend = DCV_end(vec); + const DeltaChunk* dcend = DCV_end(vec); for(;dc < dcend; dc++){ DC_destroy(dc); } @@ -366,11 +374,9 @@ DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) Py_ssize_t old_size = vec->size; vec->size += num_chunks; -#ifdef DEBUG for(;old_size < vec->size; ++old_size){ DC_init(DCV_get(vec, old_size), 0, 0, 0); } -#endif return &vec->mem[old_size]; } @@ -391,7 +397,7 @@ DeltaChunk* DCV_append(DeltaChunkVector* vec) // Return delta chunk being closest to the given absolute offset inline -DeltaChunk* DCV_closest_chunk(DeltaChunkVector* vec, ull ofs) +DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) { assert(vec->mem); @@ -416,16 +422,43 @@ DeltaChunk* DCV_closest_chunk(DeltaChunkVector* vec, ull ofs) return DCV_last(vec); } +// Assert the given vector has correct datachunks +void DCV_dbg_check_integrity(const DeltaChunkVector* vec) +{ + assert(!DCV_empty(vec)); + const DeltaChunk* i = vec->mem; + const DeltaChunk* end = DCV_end(vec); + + ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); + ull acc_size = 0; + for(; i < end; i++){ + acc_size += i->ts; + } + assert(acc_size == aparent_size); + + if (vec->size < 2){ + return; + } + + const DeltaChunk* endm1 = DCV_end(vec) - 1; + for(i = vec->mem; i < endm1; i++){ + const DeltaChunk* n = i+1; + assert(DC_rbound(i) == n->to); + } + +} + // Write a slice as defined by its absolute offset in bytes and its size into the given // destination. The individual chunks written will be a deep copy of the source // data chunks // TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort // of append-only memory pool would improve performance inline -void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) +void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) { + //fprintf(stderr, "Copy Slice To: src->size = %i, ofs = %i, size=%i\n", (int)src->size, (int)ofs, (int)size); assert(DCV_lbound(src) <= ofs); - assert(DCV_rbound(src) <= ofs + size); + assert((ofs + size) <= DCV_rbound(src)); DeltaChunk* cdc = DCV_closest_chunk(src, ofs); @@ -442,7 +475,7 @@ void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, u } } - DeltaChunk* vecend = DCV_end(src); + const DeltaChunk* vecend = DCV_end(src); for( ;(cdc < vecend) && size; ++cdc) { if (cdc->ts < size) { @@ -464,18 +497,22 @@ void DCV_copy_slice_to(DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, u // 'at' will be replaced by the items to insert ( special purpose ) // 'at' will be properly destroyed, but all items will just be copied bytewise // using memcpy. Hence from must just forget about them ! +// IMPORTANT: to must have an appropriate size already inline -void DCV_replace_one_by_many(DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) +void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) { + fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); assert(from->size > 1); + assert(to->size + from->size - 1 <= to->reserved_size); - DCV_reserve_memory(to, to->size + from->size - 1); // -1 because we replace at + // -1 because we replace 'at' DC_destroy(at); - to->size -= 1 + from->size; + to->size += from->size - 1; // If we are somewhere in the middle, we have to make some space if (DCV_last(to) != at) { - memmove((void*)at+from->size, (void*)(at+1), (size_t)(DCV_end(to) - (at+1))); + fprintf(stderr, "moving to %p from %p, num bytes = %i\n", at+from->size, at+1, (int)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); + memmove((void*)(at+from->size), (void*)(at+1), (size_t)(DCV_end(to) - (at+1)) * sizeof(DeltaChunk)); } // Finally copy all the items in @@ -485,22 +522,27 @@ void DCV_replace_one_by_many(DeltaChunkVector* from, DeltaChunkVector* to, Delta // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost // delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the // caller -void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) +void DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) { - DeltaChunk* dc = tdcv->mem; - DeltaChunk* end = tdcv->mem + tdcv->size; - assert(dc); + Py_ssize_t dci = 0; + Py_ssize_t iend = tdcv->size; + DeltaChunk* dc; - for (;dc < end; dc++) + DBG_check(tdcv); + DBG_check(bdcv); + + for (;dci < iend; dci++) { // Data chunks don't need processing + dc = DCV_get(tdcv, dci); if (dc->data){ continue; } // Copy Chunk Handling DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); - // assert(tmpl->size); + DBG_check(tmpl); + assert(tmpl->size); // move target bounds DeltaChunk* tdc = tmpl->mem; @@ -516,8 +558,15 @@ void DCV_connect_with_base(DeltaChunkVector* tdcv, DeltaChunkVector* bdcv, Delta DC_destroy(dc); *dc = *DCV_get(tmpl, 0); } else { + DCV_reserve_memory(tdcv, tdcv->size + tmpl->size - 1 + gDVC_grow_by); + dc = DCV_get(tdcv, dci); DCV_replace_one_by_many(tmpl, tdcv, dc); + // Compensate for us being replaced + dci += tmpl->size-1; + iend += tmpl->size-1; } + + DBG_check(tdcv); // make sure the members will not be deallocated by the list DCV_forget_members(tmpl); @@ -679,8 +728,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DCV_init(&tdcv, 0); DCV_init(&tmpl, 200); - unsigned int dsi; - PyObject* ds; + unsigned int dsi = 0; + PyObject* ds = 0; int error = 0; for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) { @@ -706,7 +755,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // parse command stream ull tbw = 0; // Amount of target bytes written - bool shared_data = dsi != 0; + bool is_shared_data = dsi != 0; bool is_first_run = dsi == 0; assert(data < dend); @@ -742,10 +791,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // What's faster DeltaChunk* dc = DCV_append(&dcv); DC_init(dc, tbw, cmd, 0); - DC_set_data(dc, data, cmd, shared_data); + DC_set_data(dc, data, cmd, is_shared_data); tbw += cmd; data += cmd; - } else { + } else { error = 1; PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); goto loop_end; From 4098056f4fe101f1e50d924ab3ba40650d563b9d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 00:14:06 +0200 Subject: [PATCH 0088/1392] Fixed terrible bug, which happened due to a change of the size of the vector, but too early actually, so a memmove would use incorrect values --- _fun.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/_fun.c b/_fun.c index fc4ee1edc..3be3bccab 100644 --- a/_fun.c +++ b/_fun.c @@ -501,22 +501,24 @@ void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull inline void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) { - fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); + //fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); assert(from->size > 1); assert(to->size + from->size - 1 <= to->reserved_size); // -1 because we replace 'at' DC_destroy(at); - to->size += from->size - 1; // If we are somewhere in the middle, we have to make some space if (DCV_last(to) != at) { - fprintf(stderr, "moving to %p from %p, num bytes = %i\n", at+from->size, at+1, (int)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); - memmove((void*)(at+from->size), (void*)(at+1), (size_t)(DCV_end(to) - (at+1)) * sizeof(DeltaChunk)); + //fprintf(stderr, "moving to %i from %i, num chunks = %i\n", (int)((at+from->size)-to->mem), (int)((at+1)-to->mem), (int)(DCV_end(to) - (at+1))); + memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); } - + // Finally copy all the items in memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); + + // FINALLY: update size + to->size += from->size - 1; } // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost From f563a9db73a24c6353794c8e093cc604be9c7d7b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 00:49:27 +0200 Subject: [PATCH 0089/1392] Fixed integrity check function, finalized code, so far it is working, and its very efficient as well as the amount of data-copies is minimized --- _fun.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/_fun.c b/_fun.c index 3be3bccab..0b485473f 100644 --- a/_fun.c +++ b/_fun.c @@ -95,7 +95,7 @@ typedef uchar bool; const ull gDVC_grow_by = 50; #ifdef DEBUG -#define DBG_check(vec) DCV_dbg_check_integrity(vec) +#define DBG_check(vec) asser(DCV_dbg_check_integrity(vec)) #else #define DBG_check(vec) #endif @@ -165,6 +165,26 @@ ull DC_rbound(const DeltaChunk* dc) return dc->to + dc->ts; } +// Apply +inline +void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) +{ + PyObject* buffer = 0; + if (dc->data){ + buffer = PyBuffer_FromMemory((void*)dc->data, dc->ts); + } else { + buffer = PyBuffer_FromMemory((void*)(base + dc->so), dc->ts); + } + + if (PyTuple_SetItem(tmpargs, 0, buffer)){ + assert(0); + } + + // tuple steals reference, and will take care about the deallocation + PyObject_Call(writer, tmpargs, NULL); + +} + // Copy all data from src to dest, the data pointer will be copied too inline void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) @@ -423,9 +443,12 @@ DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) } // Assert the given vector has correct datachunks -void DCV_dbg_check_integrity(const DeltaChunkVector* vec) +// return 1 on success +int DCV_dbg_check_integrity(const DeltaChunkVector* vec) { - assert(!DCV_empty(vec)); + if(DCV_empty(vec)){ + return 0; + } const DeltaChunk* i = vec->mem; const DeltaChunk* end = DCV_end(vec); @@ -434,18 +457,22 @@ void DCV_dbg_check_integrity(const DeltaChunkVector* vec) for(; i < end; i++){ acc_size += i->ts; } - assert(acc_size == aparent_size); + if (acc_size != aparent_size) + return 0; if (vec->size < 2){ - return; + return 1; } const DeltaChunk* endm1 = DCV_end(vec) - 1; for(i = vec->mem; i < endm1; i++){ const DeltaChunk* n = i+1; - assert(DC_rbound(i) == n->to); + if (DC_rbound(i) != n->to){ + return 0; + } } + return 1; } // Write a slice as defined by its absolute offset in bytes and its size into the given @@ -624,10 +651,41 @@ PyObject* DCL_py_rbound(DeltaChunkList* self) return PyLong_FromUnsignedLongLong(DCL_rbound(self)); } +// Write using a write function, taking remaining bytes from a base buffer static -PyObject* DCL_apply(PyObject* self, PyObject* args) +PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { + PyObject* pybuf = 0; + PyObject* writeproc = 0; + if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ + PyErr_BadArgument(); + return NULL; + } + + if (!PyObject_CheckReadBuffer(pybuf)){ + PyErr_SetString(PyExc_ValueError, "First argument must be a buffer-compatible object, like a string, or a memory map"); + return NULL; + } + + if (!PyCallable_Check(writeproc)){ + PyErr_SetString(PyExc_ValueError, "Second argument must be a writer method with signature write(buf)"); + return NULL; + } + + const DeltaChunk* i = self->vec.mem; + const DeltaChunk* end = DCV_end(&self->vec); + + const uchar* data; + Py_ssize_t dlen; + PyObject_AsReadBuffer(pybuf, (const void**)&data, &dlen); + + PyObject* tmpargs = PyTuple_New(1); + + for(; i < end; i++){ + DC_apply(i, data, writeproc, tmpargs); + } + Py_DECREF(tmpargs); Py_RETURN_NONE; } From 64992444e9a2b2936cea4d2cba235e59e703cac9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 10:46:21 +0200 Subject: [PATCH 0090/1392] optimized reallocation count, which improves speed a little bit. Previously it would easily get into the habbit of reallocating the vector just to add a single item --- _fun.c | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/_fun.c b/_fun.c index 0b485473f..1373a5c60 100644 --- a/_fun.c +++ b/_fun.c @@ -92,10 +92,10 @@ typedef unsigned char uchar; typedef uchar bool; // Constants -const ull gDVC_grow_by = 50; +const ull gDVC_grow_by = 100; #ifdef DEBUG -#define DBG_check(vec) asser(DCV_dbg_check_integrity(vec)) +#define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) #else #define DBG_check(vec) #endif @@ -233,6 +233,8 @@ typedef struct { // Reserve enough memory to hold the given amount of delta chunks // Return 1 on success +// NOTE: added a minimum allocation to assure reallocation is not done +// just for a single additional entry. DCVs change often, and reallocs are expensive inline int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) { @@ -240,6 +242,10 @@ int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) return 1; } + if (num_dc - vec->reserved_size){ + num_dc += gDVC_grow_by; + } + #ifdef DEBUG bool was_null = vec->mem == NULL; #endif @@ -381,25 +387,6 @@ void DCV_reset(DeltaChunkVector* vec) vec->size = 0; } -// Append num-chunks to the end of the list, possibly reallocating existing ones -// Return a pointer to the first of the added items. They are already null initialized -// If num-chunks == 0, it returns the end pointer of the allocated memory -static inline -DeltaChunk* DCV_append_multiple(DeltaChunkVector* vec, uint num_chunks) -{ - if (vec->size + num_chunks > vec->reserved_size){ - DCV_grow_by(vec, (vec->size + num_chunks) - vec->reserved_size); - } - Py_FatalError("Could not allocate memory for append operation"); - Py_ssize_t old_size = vec->size; - vec->size += num_chunks; - - for(;old_size < vec->size; ++old_size){ - DC_init(DCV_get(vec, old_size), 0, 0, 0); - } - - return &vec->mem[old_size]; -} // Append one chunk to the end of the list, and return a pointer to it // It will not have been initialized ! @@ -838,6 +825,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const unsigned long rbound = cp_off + cp_size; if (rbound < cp_size || rbound > base_size){ + assert(0); break; } @@ -849,6 +837,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // NOTE: Compression only necessary for all other deltas, not // for the first one, as we will share the data. It really depends // What's faster + // Compression reduces fragmentation though, which is why we do it + // in all cases. DeltaChunk* dc = DCV_append(&dcv); DC_init(dc, tbw, cmd, 0); DC_set_data(dc, data, cmd, is_shared_data); @@ -860,7 +850,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) goto loop_end; } }// END handle command opcodes - assert(tbw == target_size); + if (tbw != target_size){ + PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); + error = 1; + } if (!is_first_run){ DCV_connect_with_base(&tdcv, &dcv, &tmpl); From a7253b8d08ffc69bf3afdec98619044c84c64f75 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 12:19:06 +0200 Subject: [PATCH 0091/1392] implemented memory compression, but got evil memory bug once again ... probably its just as stupid as previously --- _fun.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/_fun.c b/_fun.c index 1373a5c60..cf1f5b629 100644 --- a/_fun.c +++ b/_fun.c @@ -159,6 +159,16 @@ void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared } +// Make the given data our own. It is assumed to have the size stored in our instance +// and will be managed by us. +inline +void DC_set_data_with_ownership(DeltaChunk* dc, const uchar* data) +{ + assert(data); + DC_deallocate_data(dc); + dc->data = data; +} + inline ull DC_rbound(const DeltaChunk* dc) { @@ -214,7 +224,6 @@ void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull siz if (src->data){ DC_set_data(dest, src->data + ofs, size, 0); } else { - dest->data = NULL; dest->data_shared = 0; } } @@ -825,6 +834,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const unsigned long rbound = cp_off + cp_size; if (rbound < cp_size || rbound > base_size){ + // this really shouldn't happen + error = 1; assert(0); break; } @@ -834,16 +845,53 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } else if (cmd) { // TODO: Compress nodes by parsing them in advance - // NOTE: Compression only necessary for all other deltas, not - // for the first one, as we will share the data. It really depends - // What's faster // Compression reduces fragmentation though, which is why we do it // in all cases. - DeltaChunk* dc = DCV_append(&dcv); - DC_init(dc, tbw, cmd, 0); - DC_set_data(dc, data, cmd, is_shared_data); - tbw += cmd; + const uchar* add_start = data - 1; + const uchar* add_end = dend; + ull num_bytes = cmd; data += cmd; + ull num_chunks = 1; + while (data < dend){ + fprintf(stderr, "looping\n"); + const char c = *data; + if (c & 0x80){ + add_end = data; + break; + } else { + num_chunks += 1; + data += c + 1; // advance by 1 to skip add cmd + num_bytes += c; + } + } + + fprintf(stderr, "add bytes = %i\n", (int)num_bytes); + #ifdef DEBUG + assert(add_end - add_start > 0); + if (num_chunks > 1){ + fprintf(stderr, "Compression worked, got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); + } + #endif + + DeltaChunk* dc = DCV_append(&dcv); + DC_init(dc, tbw, num_bytes, 0); + + // gather the data, or (possibly) share single blocks + if (num_chunks > 1){ + uchar* dcdata = PyMem_Malloc(num_bytes); + while (add_start < add_end){ + const char bytes = *add_start++; + fprintf(stderr, "Copying %i bytes\n", bytes); + memcpy((void*)dcdata, (void*)add_start, bytes); + dcdata += bytes; + add_start += bytes; + } + DC_set_data_with_ownership(dc, dcdata); + } else { + DC_set_data(dc, data - cmd, cmd, is_shared_data); + } + + tbw += num_bytes; } else { error = 1; PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); From bd01f6932c12fe2b7010884d339cc4831cc7ec59 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 13:27:38 +0200 Subject: [PATCH 0092/1392] Fixed memory bug, it was a small tiny thing, as well as stupid. --- _fun.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/_fun.c b/_fun.c index cf1f5b629..247008682 100644 --- a/_fun.c +++ b/_fun.c @@ -844,32 +844,32 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) tbw += cp_size; } else if (cmd) { - // TODO: Compress nodes by parsing them in advance // Compression reduces fragmentation though, which is why we do it // in all cases. + // It makes the more sense the more consecutive add-chunks we have, + // its more likely in big deltas, for big binary files const uchar* add_start = data - 1; const uchar* add_end = dend; ull num_bytes = cmd; data += cmd; ull num_chunks = 1; while (data < dend){ - fprintf(stderr, "looping\n"); + //while (0){ const char c = *data; if (c & 0x80){ add_end = data; break; } else { - num_chunks += 1; - data += c + 1; // advance by 1 to skip add cmd + data += 1 + c; // advance by 1 to skip add cmd num_bytes += c; + num_chunks += 1; } } - fprintf(stderr, "add bytes = %i\n", (int)num_bytes); #ifdef DEBUG assert(add_end - add_start > 0); if (num_chunks > 1){ - fprintf(stderr, "Compression worked, got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); + fprintf(stderr, "Compression: got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); } #endif @@ -881,12 +881,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) uchar* dcdata = PyMem_Malloc(num_bytes); while (add_start < add_end){ const char bytes = *add_start++; - fprintf(stderr, "Copying %i bytes\n", bytes); memcpy((void*)dcdata, (void*)add_start, bytes); dcdata += bytes; add_start += bytes; } - DC_set_data_with_ownership(dc, dcdata); + DC_set_data_with_ownership(dc, dcdata-num_bytes); } else { DC_set_data(dc, data - cmd, cmd, is_shared_data); } From 5442e4aec5741d5b346ab9a7cd091976a8f5e9f4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 13:48:44 +0200 Subject: [PATCH 0093/1392] Put delta-apply code into separate function. Would have preferred to to have just one dynamic module, lets see whether includes are possible --- Makefile | 3 + _delta_apply.c | 903 +++++++++++++++++++++++++++++++++++++++++++++++++ _fun.c | 885 ------------------------------------------------ fun.py | 2 +- setup.py | 5 +- 5 files changed, 911 insertions(+), 887 deletions(-) create mode 100644 _delta_apply.c diff --git a/Makefile b/Makefile index 190a66b03..e65c55a6d 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,9 @@ build:: $(SETUP) build $(SETUP) build_ext -i +build_ext:: + $(SETUP) build_ext -i + install:: $(SETUP) install diff --git a/_delta_apply.c b/_delta_apply.c new file mode 100644 index 000000000..40ea60c56 --- /dev/null +++ b/_delta_apply.c @@ -0,0 +1,903 @@ +#include +#include +#include +#include +#include +#include + +typedef unsigned long long ull; +typedef unsigned int uint; +typedef unsigned char uchar; +typedef uchar bool; + +// Constants +const ull gDVC_grow_by = 100; + +#ifdef DEBUG +#define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) +#else +#define DBG_check(vec) +#endif + +// DELTA CHUNK +//////////////// +// Internal Delta Chunk Objects +typedef struct { + ull to; + ull ts; + ull so; + const uchar* data; + bool data_shared; +} DeltaChunk; + +inline +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) +{ + dc->to = to; + dc->ts = ts; + dc->so = so; + dc->data = NULL; + dc->data_shared = 0; +} + +inline +void DC_deallocate_data(DeltaChunk* dc) +{ + if (!dc->data_shared && dc->data){ + PyMem_Free((void*)dc->data); + } + dc->data = NULL; +} + +inline +void DC_destroy(DeltaChunk* dc) +{ + DC_deallocate_data(dc); +} + +// Store a copy of data in our instance. If shared is 1, the data will be shared, +// hence it will only be stored, but the memory will not be touched, or copied. +inline +void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared) +{ + DC_deallocate_data(dc); + + if (data == 0){ + dc->data = NULL; + dc->data_shared = 0; + return; + } + + dc->data_shared = shared; + if (shared){ + dc->data = data; + } else { + dc->data = (uchar*)PyMem_Malloc(dlen); + memcpy((void*)dc->data, (void*)data, dlen); + } + +} + +// Make the given data our own. It is assumed to have the size stored in our instance +// and will be managed by us. +inline +void DC_set_data_with_ownership(DeltaChunk* dc, const uchar* data) +{ + assert(data); + DC_deallocate_data(dc); + dc->data = data; +} + +inline +ull DC_rbound(const DeltaChunk* dc) +{ + return dc->to + dc->ts; +} + +// Apply +inline +void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) +{ + PyObject* buffer = 0; + if (dc->data){ + buffer = PyBuffer_FromMemory((void*)dc->data, dc->ts); + } else { + buffer = PyBuffer_FromMemory((void*)(base + dc->so), dc->ts); + } + + if (PyTuple_SetItem(tmpargs, 0, buffer)){ + assert(0); + } + + // tuple steals reference, and will take care about the deallocation + PyObject_Call(writer, tmpargs, NULL); + +} + +// Copy all data from src to dest, the data pointer will be copied too +inline +void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) +{ + dest->to = src->to; + dest->ts = src->ts; + dest->so = src->so; + dest->data_shared = 0; + dest->data = NULL; + + DC_set_data(dest, src->data, src->ts, 0); +} + +// Copy all data with the given offset and size. The source offset, as well +// as the data will be truncated accordingly +inline +void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) +{ + assert(size <= src->ts); + assert(src->to + ofs + size <= DC_rbound(src)); + + dest->to = src->to + ofs; + dest->ts = size; + dest->so = src->so + ofs; + dest->data = NULL; + + if (src->data){ + DC_set_data(dest, src->data + ofs, size, 0); + } else { + dest->data_shared = 0; + } +} + + +// DELTA CHUNK VECTOR +///////////////////// + +typedef struct { + DeltaChunk* mem; // Memory + Py_ssize_t size; // Size in DeltaChunks + Py_ssize_t reserved_size; // Reserve in DeltaChunks +} DeltaChunkVector; + + + +// Reserve enough memory to hold the given amount of delta chunks +// Return 1 on success +// NOTE: added a minimum allocation to assure reallocation is not done +// just for a single additional entry. DCVs change often, and reallocs are expensive +inline +int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) +{ + if (num_dc <= vec->reserved_size){ + return 1; + } + + if (num_dc - vec->reserved_size){ + num_dc += gDVC_grow_by; + } + +#ifdef DEBUG + bool was_null = vec->mem == NULL; +#endif + + if (vec->mem == NULL){ + vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaChunk)); + } else { + vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaChunk)); + } + + if (vec->mem == NULL){ + Py_FatalError("Could not allocate memory for append operation"); + } + + vec->reserved_size = num_dc; + +#ifdef DEBUG + const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; + if (!was_null) + format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; + fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaChunk)), vec->mem, (int)vec->reserved_size); +#endif + + return vec->mem != NULL; +} + +/* +Grow the delta chunk list by the given amount of bytes. +This may trigger a realloc, but will do nothing if the reserved size is already +large enough. +Return 1 on success, 0 on failure +*/ +inline +int DCV_grow_by(DeltaChunkVector* vec, uint num_dc) +{ + return DCV_reserve_memory(vec, vec->reserved_size + num_dc); +} + +int DCV_init(DeltaChunkVector* vec, ull initial_size) +{ + vec->mem = NULL; + vec->size = 0; + vec->reserved_size = 0; + + return DCV_grow_by(vec, initial_size); +} + +inline +ull DCV_len(const DeltaChunkVector* vec) +{ + return vec->size; +} + +inline +ull DCV_lbound(const DeltaChunkVector* vec) +{ + assert(vec->size && vec->mem); + return vec->mem->to; +} + +// Return item at index +inline +DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) +{ + assert(i < vec->size && vec->mem); + return &vec->mem[i]; +} + +// Return last item +inline +DeltaChunk* DCV_last(const DeltaChunkVector* vec) +{ + return DCV_get(vec, vec->size-1); +} + +inline +ull DCV_rbound(const DeltaChunkVector* vec) +{ + return DC_rbound(DCV_last(vec)); +} + +inline +int DCV_empty(const DeltaChunkVector* vec) +{ + return vec->size == 0; +} + +// Return end pointer of the vector +inline +const DeltaChunk* DCV_end(const DeltaChunkVector* vec) +{ + assert(!DCV_empty(vec)); + return vec->mem + vec->size; +} + +void DCV_destroy(DeltaChunkVector* vec) +{ + if (vec->mem){ +#ifdef DEBUG + fprintf(stderr, "Freeing %p\n", (void*)vec->mem); +#endif + + const DeltaChunk* end = &vec->mem[vec->size]; + DeltaChunk* i; + for(i = vec->mem; i < end; i++){ + DC_destroy(i); + } + + PyMem_Free(vec->mem); + vec->size = 0; + vec->reserved_size = 0; + vec->mem = 0; + } +} + +// Reset this vector so that its existing memory can be filled again. +// Memory will be kept, but not cleaned up +inline +void DCV_forget_members(DeltaChunkVector* vec) +{ + vec->size = 0; +} + +// Reset the vector so that its size will be zero, and its members will +// have been deallocated properly. +// It will keep its memory though, and hence can be filled again +inline +void DCV_reset(DeltaChunkVector* vec) +{ + if (vec->size == 0) + return; + + DeltaChunk* dc = vec->mem; + const DeltaChunk* dcend = DCV_end(vec); + for(;dc < dcend; dc++){ + DC_destroy(dc); + } + + vec->size = 0; +} + + +// Append one chunk to the end of the list, and return a pointer to it +// It will not have been initialized ! +static inline +DeltaChunk* DCV_append(DeltaChunkVector* vec) +{ + if (vec->size + 1 > vec->reserved_size){ + DCV_grow_by(vec, gDVC_grow_by); + } + + DeltaChunk* next = vec->mem + vec->size; + vec->size += 1; + return next; +} + +// Return delta chunk being closest to the given absolute offset +inline +DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) +{ + assert(vec->mem); + + ull lo = 0; + ull hi = vec->size; + ull mid; + DeltaChunk* dc; + + while (lo < hi) + { + mid = (lo + hi) / 2; + dc = vec->mem + mid; + if (dc->to > ofs){ + hi = mid; + } else if ((DC_rbound(dc) > ofs) | (dc->to == ofs)) { + return dc; + } else { + lo = mid + 1; + } + } + + return DCV_last(vec); +} + +// Assert the given vector has correct datachunks +// return 1 on success +int DCV_dbg_check_integrity(const DeltaChunkVector* vec) +{ + if(DCV_empty(vec)){ + return 0; + } + const DeltaChunk* i = vec->mem; + const DeltaChunk* end = DCV_end(vec); + + ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); + ull acc_size = 0; + for(; i < end; i++){ + acc_size += i->ts; + } + if (acc_size != aparent_size) + return 0; + + if (vec->size < 2){ + return 1; + } + + const DeltaChunk* endm1 = DCV_end(vec) - 1; + for(i = vec->mem; i < endm1; i++){ + const DeltaChunk* n = i+1; + if (DC_rbound(i) != n->to){ + return 0; + } + } + + return 1; +} + +// Write a slice as defined by its absolute offset in bytes and its size into the given +// destination. The individual chunks written will be a deep copy of the source +// data chunks +// TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort +// of append-only memory pool would improve performance +inline +void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) +{ + //fprintf(stderr, "Copy Slice To: src->size = %i, ofs = %i, size=%i\n", (int)src->size, (int)ofs, (int)size); + assert(DCV_lbound(src) <= ofs); + assert((ofs + size) <= DCV_rbound(src)); + + DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + + // partial overlap + if (cdc->to != ofs) { + DeltaChunk* destc = DCV_append(dest); + const ull relofs = ofs - cdc->to; + DC_offset_copy_to(cdc, destc, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); + cdc += 1; + size -= destc->ts; + + if (size == 0){ + return; + } + } + + const DeltaChunk* vecend = DCV_end(src); + for( ;(cdc < vecend) && size; ++cdc) + { + if (cdc->ts < size) { + DC_copy_to(cdc, DCV_append(dest)); + size -= cdc->ts; + } else { + DC_offset_copy_to(cdc, DCV_append(dest), 0, size); + size = 0; + break; + } + } + + assert(size == 0); +} + + +// Insert all chunks in 'from' to 'to', starting at the delta chunk named 'at' which +// originates in to +// 'at' will be replaced by the items to insert ( special purpose ) +// 'at' will be properly destroyed, but all items will just be copied bytewise +// using memcpy. Hence from must just forget about them ! +// IMPORTANT: to must have an appropriate size already +inline +void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) +{ + //fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); + assert(from->size > 1); + assert(to->size + from->size - 1 <= to->reserved_size); + + // -1 because we replace 'at' + DC_destroy(at); + + // If we are somewhere in the middle, we have to make some space + if (DCV_last(to) != at) { + //fprintf(stderr, "moving to %i from %i, num chunks = %i\n", (int)((at+from->size)-to->mem), (int)((at+1)-to->mem), (int)(DCV_end(to) - (at+1))); + memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); + } + + // Finally copy all the items in + memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); + + // FINALLY: update size + to->size += from->size - 1; +} + +// Take slices of bdcv into the corresponding area of the tdcv, which is the topmost +// delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the +// caller +void DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) +{ + Py_ssize_t dci = 0; + Py_ssize_t iend = tdcv->size; + DeltaChunk* dc; + + DBG_check(tdcv); + DBG_check(bdcv); + + for (;dci < iend; dci++) + { + // Data chunks don't need processing + dc = DCV_get(tdcv, dci); + if (dc->data){ + continue; + } + + // Copy Chunk Handling + DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); + DBG_check(tmpl); + assert(tmpl->size); + + // move target bounds + DeltaChunk* tdc = tmpl->mem; + DeltaChunk* tdcend = tmpl->mem + tmpl->size; + const ull ofs = dc->to - dc->so; + for(;tdc < tdcend; tdc++){ + tdc->to += ofs; + } + + // insert slice into our list + if (tmpl->size == 1){ + // Its not data, so destroy is not really required, anyhow ... + DC_destroy(dc); + *dc = *DCV_get(tmpl, 0); + } else { + DCV_reserve_memory(tdcv, tdcv->size + tmpl->size - 1 + gDVC_grow_by); + dc = DCV_get(tdcv, dci); + DCV_replace_one_by_many(tmpl, tdcv, dc); + // Compensate for us being replaced + dci += tmpl->size-1; + iend += tmpl->size-1; + } + + DBG_check(tdcv); + + // make sure the members will not be deallocated by the list + DCV_forget_members(tmpl); + } +} + +// DELTA CHUNK LIST (PYTHON) +///////////////////////////// + +typedef struct { + PyObject_HEAD + // ----------- + DeltaChunkVector vec; + +} DeltaChunkList; + + +static +int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) +{ + if(args && PySequence_Size(args) > 0){ + PyErr_SetString(PyExc_ValueError, "Too many arguments"); + return -1; + } + + DCV_init(&self->vec, 0); + return 0; +} + +static +void DCL_dealloc(DeltaChunkList* self) +{ + DCV_destroy(&(self->vec)); +} + +static +PyObject* DCL_len(DeltaChunkList* self) +{ + return PyLong_FromUnsignedLongLong(DCV_len(&self->vec)); +} + +static inline +ull DCL_rbound(DeltaChunkList* self) +{ + if (DCV_empty(&self->vec)) + return 0; + return DCV_rbound(&self->vec); +} + +static +PyObject* DCL_py_rbound(DeltaChunkList* self) +{ + return PyLong_FromUnsignedLongLong(DCL_rbound(self)); +} + +// Write using a write function, taking remaining bytes from a base buffer +static +PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) +{ + PyObject* pybuf = 0; + PyObject* writeproc = 0; + if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ + PyErr_BadArgument(); + return NULL; + } + + if (!PyObject_CheckReadBuffer(pybuf)){ + PyErr_SetString(PyExc_ValueError, "First argument must be a buffer-compatible object, like a string, or a memory map"); + return NULL; + } + + if (!PyCallable_Check(writeproc)){ + PyErr_SetString(PyExc_ValueError, "Second argument must be a writer method with signature write(buf)"); + return NULL; + } + + const DeltaChunk* i = self->vec.mem; + const DeltaChunk* end = DCV_end(&self->vec); + + const uchar* data; + Py_ssize_t dlen; + PyObject_AsReadBuffer(pybuf, (const void**)&data, &dlen); + + PyObject* tmpargs = PyTuple_New(1); + + for(; i < end; i++){ + DC_apply(i, data, writeproc, tmpargs); + } + + Py_DECREF(tmpargs); + Py_RETURN_NONE; +} + +static PyMethodDef DCL_methods[] = { + {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, + {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, + {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, + {NULL} /* Sentinel */ +}; + +static PyTypeObject DeltaChunkListType = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "DeltaChunkList", /*tp_name*/ + sizeof(DeltaChunkList), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)DCL_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + "Minimal Delta Chunk List",/* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + DCL_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)DCL_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + +// Makes a new copy of the DeltaChunkList - you have to do everything yourselve +// in C ... want C++ !! +DeltaChunkList* DCL_new_instance(void) +{ + DeltaChunkList* dcl = (DeltaChunkList*) PyType_GenericNew(&DeltaChunkListType, 0, 0); + assert(dcl); + + DCL_init(dcl, 0, 0); + assert(dcl->vec.size == 0); + assert(dcl->vec.mem == NULL); + return dcl; +} + +inline +ull msb_size(const uchar** datap, const uchar* top) +{ + const uchar *data = *datap; + ull cmd, size = 0; + uint i = 0; + do { + cmd = *data++; + size |= (cmd & 0x7f) << i; + i += 7; + } while (cmd & 0x80 && data < top); + *datap = data; + return size; +} + +static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) +{ + // obtain iterator + PyObject* stream_iter = 0; + if (!PyIter_Check(dstreams)){ + stream_iter = PyObject_GetIter(dstreams); + if (!stream_iter){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't obtain iterator for streams"); + return NULL; + } + } else { + stream_iter = dstreams; + } + + DeltaChunkVector dcv; + DeltaChunkVector tdcv; + DeltaChunkVector tmpl; + DCV_init(&dcv, 100); // should be enough to keep the average text file + DCV_init(&tdcv, 0); + DCV_init(&tmpl, 200); + + unsigned int dsi = 0; + PyObject* ds = 0; + int error = 0; + for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) + { + PyObject* db = PyObject_CallMethod(ds, "read", 0); + if (!PyObject_CheckReadBuffer(db)){ + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); + goto loop_end; + } + + const uchar* data; + Py_ssize_t dlen; + PyObject_AsReadBuffer(db, (const void**)&data, &dlen); + const uchar* dend = data + dlen; + + // read header + const ull base_size = msb_size(&data, dend); + const ull target_size = msb_size(&data, dend); + + // estimate number of ops - assume one third adds, half two byte (size+offset) copies + const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); + DCV_reserve_memory(&dcv, approx_num_cmds); + + // parse command stream + ull tbw = 0; // Amount of target bytes written + bool is_shared_data = dsi != 0; + bool is_first_run = dsi == 0; + + assert(data < dend); + while (data < dend) + { + const char cmd = *data++; + + if (cmd & 0x80) + { + unsigned long cp_off = 0, cp_size = 0; + if (cmd & 0x01) cp_off = *data++; + if (cmd & 0x02) cp_off |= (*data++ << 8); + if (cmd & 0x04) cp_off |= (*data++ << 16); + if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); + if (cmd & 0x10) cp_size = *data++; + if (cmd & 0x20) cp_size |= (*data++ << 8); + if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cp_size == 0) cp_size = 0x10000; + + const unsigned long rbound = cp_off + cp_size; + if (rbound < cp_size || + rbound > base_size){ + // this really shouldn't happen + error = 1; + assert(0); + break; + } + + DC_init(DCV_append(&dcv), tbw, cp_size, cp_off); + tbw += cp_size; + + } else if (cmd) { + // Compression reduces fragmentation though, which is why we do it + // in all cases. + // It makes the more sense the more consecutive add-chunks we have, + // its more likely in big deltas, for big binary files + const uchar* add_start = data - 1; + const uchar* add_end = dend; + ull num_bytes = cmd; + data += cmd; + ull num_chunks = 1; + while (data < dend){ + //while (0){ + const char c = *data; + if (c & 0x80){ + add_end = data; + break; + } else { + data += 1 + c; // advance by 1 to skip add cmd + num_bytes += c; + num_chunks += 1; + } + } + + #ifdef DEBUG + assert(add_end - add_start > 0); + if (num_chunks > 1){ + fprintf(stderr, "Compression: got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); + } + #endif + + DeltaChunk* dc = DCV_append(&dcv); + DC_init(dc, tbw, num_bytes, 0); + + // gather the data, or (possibly) share single blocks + if (num_chunks > 1){ + uchar* dcdata = PyMem_Malloc(num_bytes); + while (add_start < add_end){ + const char bytes = *add_start++; + memcpy((void*)dcdata, (void*)add_start, bytes); + dcdata += bytes; + add_start += bytes; + } + DC_set_data_with_ownership(dc, dcdata-num_bytes); + } else { + DC_set_data(dc, data - cmd, cmd, is_shared_data); + } + + tbw += num_bytes; + } else { + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + goto loop_end; + } + }// END handle command opcodes + if (tbw != target_size){ + PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); + error = 1; + } + + if (!is_first_run){ + DCV_connect_with_base(&tdcv, &dcv, &tmpl); + } + + if (is_first_run){ + tdcv = dcv; + // wipe out dcv without destroying the members, get its own memory + DCV_init(&dcv, tdcv.size); + } else { + // destroy members, but keep memory + DCV_reset(&dcv); + } + +loop_end: + // perform cleanup + Py_DECREF(ds); + Py_DECREF(db); + + if (error){ + break; + } + }// END for each stream object + + if (dsi == 0 && ! error){ + PyErr_SetString(PyExc_ValueError, "No streams provided"); + } + + if (stream_iter != dstreams){ + Py_DECREF(stream_iter); + } + + DCV_destroy(&tmpl); + if (dsi > 1){ + // otherwise dcv equals tcl + DCV_destroy(&dcv); + } + + // Return the actual python object - its just a container + DeltaChunkList* dcl = DCL_new_instance(); + if (!dcl){ + PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); + // Otherwise tdcv would be deallocated by the chunk list + DCV_destroy(&tdcv); + error = 1; + } else { + // Plain copy, don't deallocate + dcl->vec = tdcv; + } + + if (error){ + // Will dealloc tdcv + Py_XDECREF(dcl); + return NULL; + } + + return (PyObject*)dcl; +} + +static PyMethodDef py_fun[] = { + { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, + { NULL, NULL, 0, NULL } +}; + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif +PyMODINIT_FUNC init_delta_apply(void) +{ + PyObject *m; + + if (PyType_Ready(&DeltaChunkListType) < 0) + return; + + m = Py_InitModule3("_delta_apply", py_fun, NULL); + if (m == NULL) + return; + + Py_INCREF(&DeltaChunkListType); + PyModule_AddObject(m, "DeltaChunkList", (PyObject *)&DeltaChunkListType); +} diff --git a/_fun.c b/_fun.c index 247008682..1881bfb05 100644 --- a/_fun.c +++ b/_fun.c @@ -1,9 +1,4 @@ #include -#include -#include -#include -#include -#include static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -86,883 +81,8 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) } -typedef unsigned long long ull; -typedef unsigned int uint; -typedef unsigned char uchar; -typedef uchar bool; - -// Constants -const ull gDVC_grow_by = 100; - -#ifdef DEBUG -#define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) -#else -#define DBG_check(vec) -#endif - -// DELTA CHUNK -//////////////// -// Internal Delta Chunk Objects -typedef struct { - ull to; - ull ts; - ull so; - const uchar* data; - bool data_shared; -} DeltaChunk; - -inline -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) -{ - dc->to = to; - dc->ts = ts; - dc->so = so; - dc->data = NULL; - dc->data_shared = 0; -} - -inline -void DC_deallocate_data(DeltaChunk* dc) -{ - if (!dc->data_shared && dc->data){ - PyMem_Free((void*)dc->data); - } - dc->data = NULL; -} - -inline -void DC_destroy(DeltaChunk* dc) -{ - DC_deallocate_data(dc); -} - -// Store a copy of data in our instance. If shared is 1, the data will be shared, -// hence it will only be stored, but the memory will not be touched, or copied. -inline -void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared) -{ - DC_deallocate_data(dc); - - if (data == 0){ - dc->data = NULL; - dc->data_shared = 0; - return; - } - - dc->data_shared = shared; - if (shared){ - dc->data = data; - } else { - dc->data = (uchar*)PyMem_Malloc(dlen); - memcpy((void*)dc->data, (void*)data, dlen); - } - -} - -// Make the given data our own. It is assumed to have the size stored in our instance -// and will be managed by us. -inline -void DC_set_data_with_ownership(DeltaChunk* dc, const uchar* data) -{ - assert(data); - DC_deallocate_data(dc); - dc->data = data; -} - -inline -ull DC_rbound(const DeltaChunk* dc) -{ - return dc->to + dc->ts; -} - -// Apply -inline -void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) -{ - PyObject* buffer = 0; - if (dc->data){ - buffer = PyBuffer_FromMemory((void*)dc->data, dc->ts); - } else { - buffer = PyBuffer_FromMemory((void*)(base + dc->so), dc->ts); - } - - if (PyTuple_SetItem(tmpargs, 0, buffer)){ - assert(0); - } - - // tuple steals reference, and will take care about the deallocation - PyObject_Call(writer, tmpargs, NULL); - -} - -// Copy all data from src to dest, the data pointer will be copied too -inline -void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) -{ - dest->to = src->to; - dest->ts = src->ts; - dest->so = src->so; - dest->data_shared = 0; - dest->data = NULL; - - DC_set_data(dest, src->data, src->ts, 0); -} - -// Copy all data with the given offset and size. The source offset, as well -// as the data will be truncated accordingly -inline -void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) -{ - assert(size <= src->ts); - assert(src->to + ofs + size <= DC_rbound(src)); - - dest->to = src->to + ofs; - dest->ts = size; - dest->so = src->so + ofs; - dest->data = NULL; - - if (src->data){ - DC_set_data(dest, src->data + ofs, size, 0); - } else { - dest->data_shared = 0; - } -} - - -// DELTA CHUNK VECTOR -///////////////////// - -typedef struct { - DeltaChunk* mem; // Memory - Py_ssize_t size; // Size in DeltaChunks - Py_ssize_t reserved_size; // Reserve in DeltaChunks -} DeltaChunkVector; - - - -// Reserve enough memory to hold the given amount of delta chunks -// Return 1 on success -// NOTE: added a minimum allocation to assure reallocation is not done -// just for a single additional entry. DCVs change often, and reallocs are expensive -inline -int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) -{ - if (num_dc <= vec->reserved_size){ - return 1; - } - - if (num_dc - vec->reserved_size){ - num_dc += gDVC_grow_by; - } - -#ifdef DEBUG - bool was_null = vec->mem == NULL; -#endif - - if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaChunk)); - } else { - vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaChunk)); - } - - if (vec->mem == NULL){ - Py_FatalError("Could not allocate memory for append operation"); - } - - vec->reserved_size = num_dc; - -#ifdef DEBUG - const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; - if (!was_null) - format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; - fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaChunk)), vec->mem, (int)vec->reserved_size); -#endif - - return vec->mem != NULL; -} - -/* -Grow the delta chunk list by the given amount of bytes. -This may trigger a realloc, but will do nothing if the reserved size is already -large enough. -Return 1 on success, 0 on failure -*/ -inline -int DCV_grow_by(DeltaChunkVector* vec, uint num_dc) -{ - return DCV_reserve_memory(vec, vec->reserved_size + num_dc); -} - -int DCV_init(DeltaChunkVector* vec, ull initial_size) -{ - vec->mem = NULL; - vec->size = 0; - vec->reserved_size = 0; - - return DCV_grow_by(vec, initial_size); -} - -inline -ull DCV_len(const DeltaChunkVector* vec) -{ - return vec->size; -} - -inline -ull DCV_lbound(const DeltaChunkVector* vec) -{ - assert(vec->size && vec->mem); - return vec->mem->to; -} - -// Return item at index -inline -DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) -{ - assert(i < vec->size && vec->mem); - return &vec->mem[i]; -} - -// Return last item -inline -DeltaChunk* DCV_last(const DeltaChunkVector* vec) -{ - return DCV_get(vec, vec->size-1); -} - -inline -ull DCV_rbound(const DeltaChunkVector* vec) -{ - return DC_rbound(DCV_last(vec)); -} - -inline -int DCV_empty(const DeltaChunkVector* vec) -{ - return vec->size == 0; -} - -// Return end pointer of the vector -inline -const DeltaChunk* DCV_end(const DeltaChunkVector* vec) -{ - assert(!DCV_empty(vec)); - return vec->mem + vec->size; -} - -void DCV_destroy(DeltaChunkVector* vec) -{ - if (vec->mem){ -#ifdef DEBUG - fprintf(stderr, "Freeing %p\n", (void*)vec->mem); -#endif - - const DeltaChunk* end = &vec->mem[vec->size]; - DeltaChunk* i; - for(i = vec->mem; i < end; i++){ - DC_destroy(i); - } - - PyMem_Free(vec->mem); - vec->size = 0; - vec->reserved_size = 0; - vec->mem = 0; - } -} - -// Reset this vector so that its existing memory can be filled again. -// Memory will be kept, but not cleaned up -inline -void DCV_forget_members(DeltaChunkVector* vec) -{ - vec->size = 0; -} - -// Reset the vector so that its size will be zero, and its members will -// have been deallocated properly. -// It will keep its memory though, and hence can be filled again -inline -void DCV_reset(DeltaChunkVector* vec) -{ - if (vec->size == 0) - return; - - DeltaChunk* dc = vec->mem; - const DeltaChunk* dcend = DCV_end(vec); - for(;dc < dcend; dc++){ - DC_destroy(dc); - } - - vec->size = 0; -} - - -// Append one chunk to the end of the list, and return a pointer to it -// It will not have been initialized ! -static inline -DeltaChunk* DCV_append(DeltaChunkVector* vec) -{ - if (vec->size + 1 > vec->reserved_size){ - DCV_grow_by(vec, gDVC_grow_by); - } - - DeltaChunk* next = vec->mem + vec->size; - vec->size += 1; - return next; -} - -// Return delta chunk being closest to the given absolute offset -inline -DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) -{ - assert(vec->mem); - - ull lo = 0; - ull hi = vec->size; - ull mid; - DeltaChunk* dc; - - while (lo < hi) - { - mid = (lo + hi) / 2; - dc = vec->mem + mid; - if (dc->to > ofs){ - hi = mid; - } else if ((DC_rbound(dc) > ofs) | (dc->to == ofs)) { - return dc; - } else { - lo = mid + 1; - } - } - - return DCV_last(vec); -} - -// Assert the given vector has correct datachunks -// return 1 on success -int DCV_dbg_check_integrity(const DeltaChunkVector* vec) -{ - if(DCV_empty(vec)){ - return 0; - } - const DeltaChunk* i = vec->mem; - const DeltaChunk* end = DCV_end(vec); - - ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); - ull acc_size = 0; - for(; i < end; i++){ - acc_size += i->ts; - } - if (acc_size != aparent_size) - return 0; - - if (vec->size < 2){ - return 1; - } - - const DeltaChunk* endm1 = DCV_end(vec) - 1; - for(i = vec->mem; i < endm1; i++){ - const DeltaChunk* n = i+1; - if (DC_rbound(i) != n->to){ - return 0; - } - } - - return 1; -} - -// Write a slice as defined by its absolute offset in bytes and its size into the given -// destination. The individual chunks written will be a deep copy of the source -// data chunks -// TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort -// of append-only memory pool would improve performance -inline -void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) -{ - //fprintf(stderr, "Copy Slice To: src->size = %i, ofs = %i, size=%i\n", (int)src->size, (int)ofs, (int)size); - assert(DCV_lbound(src) <= ofs); - assert((ofs + size) <= DCV_rbound(src)); - - DeltaChunk* cdc = DCV_closest_chunk(src, ofs); - - // partial overlap - if (cdc->to != ofs) { - DeltaChunk* destc = DCV_append(dest); - const ull relofs = ofs - cdc->to; - DC_offset_copy_to(cdc, destc, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); - cdc += 1; - size -= destc->ts; - - if (size == 0){ - return; - } - } - - const DeltaChunk* vecend = DCV_end(src); - for( ;(cdc < vecend) && size; ++cdc) - { - if (cdc->ts < size) { - DC_copy_to(cdc, DCV_append(dest)); - size -= cdc->ts; - } else { - DC_offset_copy_to(cdc, DCV_append(dest), 0, size); - size = 0; - break; - } - } - - assert(size == 0); -} - - -// Insert all chunks in 'from' to 'to', starting at the delta chunk named 'at' which -// originates in to -// 'at' will be replaced by the items to insert ( special purpose ) -// 'at' will be properly destroyed, but all items will just be copied bytewise -// using memcpy. Hence from must just forget about them ! -// IMPORTANT: to must have an appropriate size already -inline -void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) -{ - //fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); - assert(from->size > 1); - assert(to->size + from->size - 1 <= to->reserved_size); - - // -1 because we replace 'at' - DC_destroy(at); - - // If we are somewhere in the middle, we have to make some space - if (DCV_last(to) != at) { - //fprintf(stderr, "moving to %i from %i, num chunks = %i\n", (int)((at+from->size)-to->mem), (int)((at+1)-to->mem), (int)(DCV_end(to) - (at+1))); - memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); - } - - // Finally copy all the items in - memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); - - // FINALLY: update size - to->size += from->size - 1; -} - -// Take slices of bdcv into the corresponding area of the tdcv, which is the topmost -// delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the -// caller -void DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) -{ - Py_ssize_t dci = 0; - Py_ssize_t iend = tdcv->size; - DeltaChunk* dc; - - DBG_check(tdcv); - DBG_check(bdcv); - - for (;dci < iend; dci++) - { - // Data chunks don't need processing - dc = DCV_get(tdcv, dci); - if (dc->data){ - continue; - } - - // Copy Chunk Handling - DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); - DBG_check(tmpl); - assert(tmpl->size); - - // move target bounds - DeltaChunk* tdc = tmpl->mem; - DeltaChunk* tdcend = tmpl->mem + tmpl->size; - const ull ofs = dc->to - dc->so; - for(;tdc < tdcend; tdc++){ - tdc->to += ofs; - } - - // insert slice into our list - if (tmpl->size == 1){ - // Its not data, so destroy is not really required, anyhow ... - DC_destroy(dc); - *dc = *DCV_get(tmpl, 0); - } else { - DCV_reserve_memory(tdcv, tdcv->size + tmpl->size - 1 + gDVC_grow_by); - dc = DCV_get(tdcv, dci); - DCV_replace_one_by_many(tmpl, tdcv, dc); - // Compensate for us being replaced - dci += tmpl->size-1; - iend += tmpl->size-1; - } - - DBG_check(tdcv); - - // make sure the members will not be deallocated by the list - DCV_forget_members(tmpl); - } -} - -// DELTA CHUNK LIST (PYTHON) -///////////////////////////// - -typedef struct { - PyObject_HEAD - // ----------- - DeltaChunkVector vec; - -} DeltaChunkList; - - -static -int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) -{ - if(args && PySequence_Size(args) > 0){ - PyErr_SetString(PyExc_ValueError, "Too many arguments"); - return -1; - } - - DCV_init(&self->vec, 0); - return 0; -} - -static -void DCL_dealloc(DeltaChunkList* self) -{ - DCV_destroy(&(self->vec)); -} - -static -PyObject* DCL_len(DeltaChunkList* self) -{ - return PyLong_FromUnsignedLongLong(DCV_len(&self->vec)); -} - -static inline -ull DCL_rbound(DeltaChunkList* self) -{ - if (DCV_empty(&self->vec)) - return 0; - return DCV_rbound(&self->vec); -} - -static -PyObject* DCL_py_rbound(DeltaChunkList* self) -{ - return PyLong_FromUnsignedLongLong(DCL_rbound(self)); -} - -// Write using a write function, taking remaining bytes from a base buffer -static -PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) -{ - PyObject* pybuf = 0; - PyObject* writeproc = 0; - if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ - PyErr_BadArgument(); - return NULL; - } - - if (!PyObject_CheckReadBuffer(pybuf)){ - PyErr_SetString(PyExc_ValueError, "First argument must be a buffer-compatible object, like a string, or a memory map"); - return NULL; - } - - if (!PyCallable_Check(writeproc)){ - PyErr_SetString(PyExc_ValueError, "Second argument must be a writer method with signature write(buf)"); - return NULL; - } - - const DeltaChunk* i = self->vec.mem; - const DeltaChunk* end = DCV_end(&self->vec); - - const uchar* data; - Py_ssize_t dlen; - PyObject_AsReadBuffer(pybuf, (const void**)&data, &dlen); - - PyObject* tmpargs = PyTuple_New(1); - - for(; i < end; i++){ - DC_apply(i, data, writeproc, tmpargs); - } - - Py_DECREF(tmpargs); - Py_RETURN_NONE; -} - -static PyMethodDef DCL_methods[] = { - {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, - {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, - {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, - {NULL} /* Sentinel */ -}; - -static PyTypeObject DeltaChunkListType = { - PyObject_HEAD_INIT(NULL) - 0, /*ob_size*/ - "DeltaChunkList", /*tp_name*/ - sizeof(DeltaChunkList), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)DCL_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - "Minimal Delta Chunk List",/* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - DCL_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)DCL_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; - - -// Makes a new copy of the DeltaChunkList - you have to do everything yourselve -// in C ... want C++ !! -DeltaChunkList* DCL_new_instance(void) -{ - DeltaChunkList* dcl = (DeltaChunkList*) PyType_GenericNew(&DeltaChunkListType, 0, 0); - assert(dcl); - - DCL_init(dcl, 0, 0); - assert(dcl->vec.size == 0); - assert(dcl->vec.mem == NULL); - return dcl; -} - -inline -ull msb_size(const uchar** datap, const uchar* top) -{ - const uchar *data = *datap; - ull cmd, size = 0; - uint i = 0; - do { - cmd = *data++; - size |= (cmd & 0x7f) << i; - i += 7; - } while (cmd & 0x80 && data < top); - *datap = data; - return size; -} - -static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) -{ - // obtain iterator - PyObject* stream_iter = 0; - if (!PyIter_Check(dstreams)){ - stream_iter = PyObject_GetIter(dstreams); - if (!stream_iter){ - PyErr_SetString(PyExc_RuntimeError, "Couldn't obtain iterator for streams"); - return NULL; - } - } else { - stream_iter = dstreams; - } - - DeltaChunkVector dcv; - DeltaChunkVector tdcv; - DeltaChunkVector tmpl; - DCV_init(&dcv, 100); // should be enough to keep the average text file - DCV_init(&tdcv, 0); - DCV_init(&tmpl, 200); - - unsigned int dsi = 0; - PyObject* ds = 0; - int error = 0; - for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) - { - PyObject* db = PyObject_CallMethod(ds, "read", 0); - if (!PyObject_CheckReadBuffer(db)){ - error = 1; - PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); - goto loop_end; - } - - const uchar* data; - Py_ssize_t dlen; - PyObject_AsReadBuffer(db, (const void**)&data, &dlen); - const uchar* dend = data + dlen; - - // read header - const ull base_size = msb_size(&data, dend); - const ull target_size = msb_size(&data, dend); - - // estimate number of ops - assume one third adds, half two byte (size+offset) copies - const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); - DCV_reserve_memory(&dcv, approx_num_cmds); - - // parse command stream - ull tbw = 0; // Amount of target bytes written - bool is_shared_data = dsi != 0; - bool is_first_run = dsi == 0; - - assert(data < dend); - while (data < dend) - { - const char cmd = *data++; - - if (cmd & 0x80) - { - unsigned long cp_off = 0, cp_size = 0; - if (cmd & 0x01) cp_off = *data++; - if (cmd & 0x02) cp_off |= (*data++ << 8); - if (cmd & 0x04) cp_off |= (*data++ << 16); - if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); - if (cmd & 0x10) cp_size = *data++; - if (cmd & 0x20) cp_size |= (*data++ << 8); - if (cmd & 0x40) cp_size |= (*data++ << 16); - if (cp_size == 0) cp_size = 0x10000; - - const unsigned long rbound = cp_off + cp_size; - if (rbound < cp_size || - rbound > base_size){ - // this really shouldn't happen - error = 1; - assert(0); - break; - } - - DC_init(DCV_append(&dcv), tbw, cp_size, cp_off); - tbw += cp_size; - - } else if (cmd) { - // Compression reduces fragmentation though, which is why we do it - // in all cases. - // It makes the more sense the more consecutive add-chunks we have, - // its more likely in big deltas, for big binary files - const uchar* add_start = data - 1; - const uchar* add_end = dend; - ull num_bytes = cmd; - data += cmd; - ull num_chunks = 1; - while (data < dend){ - //while (0){ - const char c = *data; - if (c & 0x80){ - add_end = data; - break; - } else { - data += 1 + c; // advance by 1 to skip add cmd - num_bytes += c; - num_chunks += 1; - } - } - - #ifdef DEBUG - assert(add_end - add_start > 0); - if (num_chunks > 1){ - fprintf(stderr, "Compression: got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); - } - #endif - - DeltaChunk* dc = DCV_append(&dcv); - DC_init(dc, tbw, num_bytes, 0); - - // gather the data, or (possibly) share single blocks - if (num_chunks > 1){ - uchar* dcdata = PyMem_Malloc(num_bytes); - while (add_start < add_end){ - const char bytes = *add_start++; - memcpy((void*)dcdata, (void*)add_start, bytes); - dcdata += bytes; - add_start += bytes; - } - DC_set_data_with_ownership(dc, dcdata-num_bytes); - } else { - DC_set_data(dc, data - cmd, cmd, is_shared_data); - } - - tbw += num_bytes; - } else { - error = 1; - PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); - goto loop_end; - } - }// END handle command opcodes - if (tbw != target_size){ - PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); - error = 1; - } - - if (!is_first_run){ - DCV_connect_with_base(&tdcv, &dcv, &tmpl); - } - - if (is_first_run){ - tdcv = dcv; - // wipe out dcv without destroying the members, get its own memory - DCV_init(&dcv, tdcv.size); - } else { - // destroy members, but keep memory - DCV_reset(&dcv); - } - -loop_end: - // perform cleanup - Py_DECREF(ds); - Py_DECREF(db); - - if (error){ - break; - } - }// END for each stream object - - if (dsi == 0 && ! error){ - PyErr_SetString(PyExc_ValueError, "No streams provided"); - } - - if (stream_iter != dstreams){ - Py_DECREF(stream_iter); - } - - DCV_destroy(&tmpl); - if (dsi > 1){ - // otherwise dcv equals tcl - DCV_destroy(&dcv); - } - - // Return the actual python object - its just a container - DeltaChunkList* dcl = DCL_new_instance(); - if (!dcl){ - PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); - // Otherwise tdcv would be deallocated by the chunk list - DCV_destroy(&tdcv); - error = 1; - } else { - // Plain copy, don't deallocate - dcl->vec = tdcv; - } - - if (error){ - // Will dealloc tdcv - Py_XDECREF(dcl); - return NULL; - } - - return (PyObject*)dcl; -} - static PyMethodDef py_fun[] = { { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, - { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, { NULL, NULL, 0, NULL } }; @@ -973,13 +93,8 @@ PyMODINIT_FUNC init_fun(void) { PyObject *m; - if (PyType_Ready(&DeltaChunkListType) < 0) - return; - m = Py_InitModule3("_fun", py_fun, NULL); if (m == NULL) return; - Py_INCREF(&DeltaChunkListType); - PyModule_AddObject(m, "Noddy", (PyObject *)&DeltaChunkListType); } diff --git a/fun.py b/fun.py index 038b4c761..e17460bf6 100644 --- a/fun.py +++ b/fun.py @@ -654,6 +654,6 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: # raise ImportError; # DEBUG - from _fun import connect_deltas + from _delta_apply import connect_deltas except ImportError: pass diff --git a/setup.py b/setup.py index 265156df2..7146ea833 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,10 @@ def get_data_files(self): package_data={'gitdb' : ['AUTHORS', 'README'], 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':''}, - ext_modules=[Extension('gitdb._fun', ['_fun.c'])], + ext_modules=[ + Extension('gitdb._fun', ['_fun.c']), + Extension('gitdb._delta_apply', ['_delta_apply.c']) + ], license = "BSD License", requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', From 9c5672e1b0cf56f1cf5151d60acb35d610e904a8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 13:55:22 +0200 Subject: [PATCH 0094/1392] Now building a single module called _perf which contains all the performance enhancements, which increases loadtimes, less is more --- _delta_apply.c | 22 ---------------------- _fun.c | 12 +++++++++--- fun.py | 2 +- pack.py | 2 +- setup.py | 5 +---- 5 files changed, 12 insertions(+), 31 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 40ea60c56..d81e65d15 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -879,25 +879,3 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) return (PyObject*)dcl; } -static PyMethodDef py_fun[] = { - { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, - { NULL, NULL, 0, NULL } -}; - -#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ -#define PyMODINIT_FUNC void -#endif -PyMODINIT_FUNC init_delta_apply(void) -{ - PyObject *m; - - if (PyType_Ready(&DeltaChunkListType) < 0) - return; - - m = Py_InitModule3("_delta_apply", py_fun, NULL); - if (m == NULL) - return; - - Py_INCREF(&DeltaChunkListType); - PyModule_AddObject(m, "DeltaChunkList", (PyObject *)&DeltaChunkListType); -} diff --git a/_fun.c b/_fun.c index 1881bfb05..386068577 100644 --- a/_fun.c +++ b/_fun.c @@ -1,4 +1,5 @@ #include +#include "_delta_apply.c" static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { @@ -80,21 +81,26 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) Py_RETURN_NONE; } - static PyMethodDef py_fun[] = { { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, + { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, { NULL, NULL, 0, NULL } }; #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ #define PyMODINIT_FUNC void #endif -PyMODINIT_FUNC init_fun(void) +PyMODINIT_FUNC init_perf(void) { PyObject *m; - m = Py_InitModule3("_fun", py_fun, NULL); + if (PyType_Ready(&DeltaChunkListType) < 0) + return; + + m = Py_InitModule3("_perf", py_fun, NULL); if (m == NULL) return; + Py_INCREF(&DeltaChunkListType); + PyModule_AddObject(m, "DeltaChunkList", (PyObject *)&DeltaChunkListType); } diff --git a/fun.py b/fun.py index e17460bf6..0b14f82ac 100644 --- a/fun.py +++ b/fun.py @@ -654,6 +654,6 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: # raise ImportError; # DEBUG - from _delta_apply import connect_deltas + from _perf import connect_deltas except ImportError: pass diff --git a/pack.py b/pack.py index 91323fcce..79ffcc217 100644 --- a/pack.py +++ b/pack.py @@ -24,7 +24,7 @@ ) try: - from _fun import PackIndexFile_sha_to_index + from _perf import PackIndexFile_sha_to_index except ImportError: pass # END try c module diff --git a/setup.py b/setup.py index 7146ea833..7e50e0fe7 100755 --- a/setup.py +++ b/setup.py @@ -77,10 +77,7 @@ def get_data_files(self): package_data={'gitdb' : ['AUTHORS', 'README'], 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':''}, - ext_modules=[ - Extension('gitdb._fun', ['_fun.c']), - Extension('gitdb._delta_apply', ['_delta_apply.c']) - ], + ext_modules=[Extension('gitdb._perf', ['_fun.c'])], license = "BSD License", requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', From fa03f746746df99763deb45943988d68fb450b4b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 15:51:04 +0200 Subject: [PATCH 0095/1392] Reverse Delta Application was a nice experiment, as it has one major flaw: Currently it integrates chunks from its base into the topmost delta chunk list, which causes plenty of mem-move operations. Plenty means, many many many, and its getting worse the more deltas you have of course. The algorithm was supposed to reduce the amount of memory activity, but failed at this point, making it worse than before. Probably it would just be fastest to implement the previous python algorithm, which swaps two buffers, in c --- _delta_apply.c | 11 +++++++---- stream.py | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index d81e65d15..e667a2eda 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -398,7 +398,6 @@ int DCV_dbg_check_integrity(const DeltaChunkVector* vec) inline void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) { - //fprintf(stderr, "Copy Slice To: src->size = %i, ofs = %i, size=%i\n", (int)src->size, (int)ofs, (int)size); assert(DCV_lbound(src) <= ofs); assert((ofs + size) <= DCV_rbound(src)); @@ -443,7 +442,6 @@ void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull inline void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) { - //fprintf(stderr, "Replace one by many: from->size = %i, to->size = %i, to->reserved = %i\n", (int)from->size, (int)to->size, (int)to->reserved_size); assert(from->size > 1); assert(to->size + from->size - 1 <= to->reserved_size); @@ -452,7 +450,6 @@ void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, // If we are somewhere in the middle, we have to make some space if (DCV_last(to) != at) { - //fprintf(stderr, "moving to %i from %i, num chunks = %i\n", (int)((at+from->size)-to->mem), (int)((at+1)-to->mem), (int)(DCV_end(to) - (at+1))); memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); } @@ -725,7 +722,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const ull target_size = msb_size(&data, dend); // estimate number of ops - assume one third adds, half two byte (size+offset) copies - const uint approx_num_cmds = (dlen / 3) + (((dlen / 3) * 2) / (2+2+1)); + // Assume good compression for the adds + const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); DCV_reserve_memory(&dcv, approx_num_cmds); // parse command stream @@ -825,6 +823,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DCV_connect_with_base(&tdcv, &dcv, &tmpl); } + #ifdef DEBUG + fprintf(stderr, "tdcv->size = %i, tdcv->reserved_size = %i\n", (int)tdcv.size, (int)tdcv.reserved_size); + fprintf(stderr, "dcv->size = %i, dcv->reserved_size = %i\n", (int)dcv.size, (int)dcv.reserved_size); + #endif + if (is_first_run){ tdcv = dcv; // wipe out dcv without destroying the members, get its own memory diff --git a/stream.py b/stream.py index af4591f85..8f9f9c386 100644 --- a/stream.py +++ b/stream.py @@ -337,6 +337,7 @@ def _set_cache_(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. + # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) dcl = connect_deltas(self._dstreams) # call len directly, as the (optional) c version doesn't implement the sequence From ff9c83a3fea26653d725686a692d0100efb63382 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 16:28:07 +0200 Subject: [PATCH 0096/1392] Disabled new implementation in favor of the old one - all that's needed is a c implementation of apply delta data --- _delta_apply.c | 4 ++++ stream.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index e667a2eda..51f55e892 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -450,6 +450,10 @@ void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, // If we are somewhere in the middle, we have to make some space if (DCV_last(to) != at) { + // IMPORTANT: This memmove kills the performance in case of large deltas + // Causing everything to slow down enormously. Its logical, as the memory + // gets shifted each time we insert nodes, for each chunk, for ever smaller + // chunks depending on the deltas memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); } diff --git a/stream.py b/stream.py index 8f9f9c386..2c24426c6 100644 --- a/stream.py +++ b/stream.py @@ -325,7 +325,7 @@ def __init__(self, stream_list): self._dstreams = tuple(stream_list[:-1]) self._br = 0 - def _set_cache_(self, attr): + def _set_cache_too_slow(self, attr): # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller # than X - definitely the case in python, every function call costs @@ -360,7 +360,7 @@ def _set_cache_(self, attr): self._mm_target.seek(0) - def _set_cache_brute_(self, attr): + def _set_cache_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() From 95d820239d5444d59ae67f50de715f1574c7213b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 17:38:53 +0200 Subject: [PATCH 0097/1392] apply_delta now has a C implementation, which is only 25 percent faster for small files ( the overhead of all the rest is very high ), but 4 times faster for large files, where the enormous call overhead coming in with python really starts to show --- _delta_apply.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ _fun.c | 3 ++- stream.py | 17 ++++++------ 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 51f55e892..dfb0a94c3 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -886,3 +886,73 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) return (PyObject*)dcl; } + +// Write using a write function, taking remaining bytes from a base buffer +// replaces the corresponding method in python +static +PyObject* apply_delta(PyObject* self, PyObject* args) +{ + PyObject* pybbuf = 0; + PyObject* pydbuf = 0; + PyObject* pytbuf = 0; + if (!PyArg_ParseTuple(args, "OOO", &pybbuf, &pydbuf, &pytbuf)){ + PyErr_BadArgument(); + return NULL; + } + + PyObject* objects[] = { pybbuf, pydbuf, pytbuf }; + assert(sizeof(objects) / sizeof(PyObject*) == 3); + + uint i; + for(i = 0; i < 3; i++){ + if (!PyObject_CheckReadBuffer(objects[i])){ + PyErr_SetString(PyExc_ValueError, "Argument must be a buffer-compatible object, like a string, or a memory map"); + return NULL; + } + } + + Py_ssize_t lbbuf; Py_ssize_t ldbuf; Py_ssize_t ltbuf; + const uchar* bbuf; const uchar* dbuf; + uchar* tbuf; + PyObject_AsReadBuffer(pybbuf, (const void**)(&bbuf), &lbbuf); + PyObject_AsReadBuffer(pydbuf, (const void**)(&dbuf), &ldbuf); + + if (PyObject_AsWriteBuffer(pytbuf, (void**)(&tbuf), <buf)){ + PyErr_SetString(PyExc_ValueError, "Argument 3 must be a writable buffer"); + return NULL; + } + + const uchar* data = dbuf; + const uchar* dend = dbuf + ldbuf; + + while (data < dend) + { + const char cmd = *data++; + + if (cmd & 0x80) + { + unsigned long cp_off = 0, cp_size = 0; + if (cmd & 0x01) cp_off = *data++; + if (cmd & 0x02) cp_off |= (*data++ << 8); + if (cmd & 0x04) cp_off |= (*data++ << 16); + if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); + if (cmd & 0x10) cp_size = *data++; + if (cmd & 0x20) cp_size |= (*data++ << 8); + if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cp_size == 0) cp_size = 0x10000; + + memcpy(tbuf, bbuf + cp_off, cp_size); + tbuf += cp_size; + + } else if (cmd) { + memcpy(tbuf, data, cmd); + tbuf += cmd; + data += cmd; + } else { + PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + return NULL; + } + }// END handle command opcodes + + Py_RETURN_NONE; +} diff --git a/_fun.c b/_fun.c index 386068577..befee4ec4 100644 --- a/_fun.c +++ b/_fun.c @@ -83,7 +83,8 @@ static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) static PyMethodDef py_fun[] = { { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, - { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "TODO" }, + { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "See python implementation" }, + { "apply_delta", (PyCFunction)apply_delta, METH_VARARGS, "See python implementation" }, { NULL, NULL, 0, NULL } }; diff --git a/stream.py b/stream.py index 2c24426c6..cedb70cf3 100644 --- a/stream.py +++ b/stream.py @@ -22,6 +22,11 @@ zlib ) +try: + from _perf import apply_delta as c_apply_delta +except ImportError: + pass + __all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader') @@ -413,7 +418,10 @@ def _set_cache_(self, attr): stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) ####################################################################### - apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) + if 'c_apply_delta' in globals(): + c_apply_delta(bbuf, ddata, tbuf); + else: + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) ####################################################################### # finally, swap out source and target buffers. The target is now the @@ -430,13 +438,6 @@ def _set_cache_(self, attr): self._mm_target = bbuf self._size = final_target_size - # TODO: Once that works, figure out the ordering of the opcodes. If they - # are always in-order/sequential, an alternate implementation could - # use stream access only. Of course this would mean we would read - # all deltas in advance, analyse the opcode ranges to determine a final - # concatenated opcode list which indicates what to copy from which delta - # to which position. This preprocessing would allow true streaming - def read(self, count=0): bl = self._size - self._br # bytes left if count < 1 or count > bl: From f3284c1edf3670dca0cf7fb11ad50a36fe53e782 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 19:21:48 +0200 Subject: [PATCH 0098/1392] Integrated new algorithm into the stream class, it will now be chosen depending on the context to figure out which one to use. For some reason, the c version that is slow for big files really rocks when its about small files. Its better than the respective c implementation of the normal delta apply --- stream.py | 23 ++++++++++++++++++++++- test/performance/test_pack.py | 33 +++++++++++---------------------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/stream.py b/stream.py index cedb70cf3..f5e05e114 100644 --- a/stream.py +++ b/stream.py @@ -22,8 +22,10 @@ zlib ) +has_perf_mod = False try: from _perf import apply_delta as c_apply_delta + has_perf_mod = True except ImportError: pass @@ -330,7 +332,7 @@ def __init__(self, stream_list): self._dstreams = tuple(stream_list[:-1]) self._br = 0 - def _set_cache_too_slow(self, attr): + def _set_cache_too_slow_without_c(self, attr): # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller # than X - definitely the case in python, every function call costs @@ -366,6 +368,15 @@ def _set_cache_too_slow(self, attr): self._mm_target.seek(0) def _set_cache_(self, attr): + """Determine which version to use depending on the configuration of the deltas + :note: we are only called if we have the performance module""" + # otherwise it depends on the amount of memory to shift around + if len(self._dstreams) > 1 and self._bstream.size < 150000: + return self._set_cache_too_slow_without_c(attr) + else: + return self._set_cache_brute_(attr) + + def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" buffer_info_list = list() @@ -438,6 +449,13 @@ def _set_cache_(self, attr): self._mm_target = bbuf self._size = final_target_size + + #{ Configuration + if not has_perf_mod: + _set_cache_ = _set_cache_brute_ + + #} END configuration + def read(self, count=0): bl = self._size - self._br # bytes left if count < 1 or count > bl: @@ -654,4 +672,7 @@ def close(self): def write(self, data): return len(data) + #} END W streams + + diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index af468b0be..f9169ffb0 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -25,29 +25,18 @@ def test_pack_random_access(self): # sha lookup: best-case and worst case access pdb_pack_info = pdb._pack_info - access_times = list() - for rand in range(2): - if rand: - random.shuffle(sha_list) - # END shuffle shas - st = time() - for sha in sha_list: - pdb_pack_info(sha) - # END for each sha to look up - elapsed = time() - st - access_times.append(elapsed) - - # discard cache - del(pdb._entities) - pdb.entities() - print >> sys.stderr, "PDB: looked up %i sha in %i packs (random=%i) in %f s ( %f shas/s )" % (ns, len(pdb.entities()), rand, elapsed, ns / elapsed) - # END for each random mode - elapsed_order, elapsed_rand = access_times - - # well, its never really sequencial regarding the memory patterns, but it - # shows how well the prioriy cache performs - print >> sys.stderr, "PDB: sequential access is %f %% faster than random-access" % (100 - ((elapsed_order / elapsed_rand) * 100)) + # END shuffle shas + st = time() + for sha in sha_list: + pdb_pack_info(sha) + # END for each sha to look up + elapsed = time() - st + # discard cache + del(pdb._entities) + pdb.entities() + print >> sys.stderr, "PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed) + # END for each random mode # query info and streams only max_items = 10000 # can wait longer when testing memory From f59c58d9478ddc49964b5bcd0711d06a957e4680 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 13 Oct 2010 21:46:05 +0200 Subject: [PATCH 0099/1392] Added new stream type which will request its size from its stream. This triggers full decompression, currently, but allows work to be delayed even further.If people want a stream, they usually read it anyway. Then it doesn't matter which attriubute they query first --- base.py | 17 +++++++++++++++++ pack.py | 8 ++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/base.py b/base.py index 938a09242..d0bcc0866 100644 --- a/base.py +++ b/base.py @@ -135,6 +135,23 @@ def read(self, size=-1): @property def stream(self): return self[3] + + #} END stream reader interface + + +class ODeltaStream(OStream): + """Uses size info of its stream, delaying reads""" + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + #{ Stream Reader Interface + + @property + def size(self): + return self[3].size + #} END stream reader interface diff --git a/pack.py b/pack.py index 79ffcc217..30da52c63 100644 --- a/pack.py +++ b/pack.py @@ -34,6 +34,7 @@ OStream, OPackInfo, OPackStream, + ODeltaStream, ODeltaPackInfo, ODeltaPackStream, ) @@ -584,14 +585,9 @@ def _object(self, sha, as_stream, index=-1): # To prevent it from applying the deltas when querying the size, # we extract it from the delta stream ourselves streams = self.collect_streams_at_offset(offset) - buf = streams[0].read(512) - offset, src_size = msb_size(buf) - offset, target_size = msb_size(buf, offset) - - streams[0].stream.seek(0) # assure it can be read by the delta reader dstream = DeltaApplyReader.new(streams) - return OStream(sha, dstream.type, target_size, dstream) + return ODeltaStream(sha, dstream.type, None, dstream) else: if type_id not in delta_types: return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) From dcdd0fd9aa8aea6989cbf1530e73c6c86fc95761 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 19:17:22 +0200 Subject: [PATCH 0100/1392] Added initial version of a document to show possible ways to stream delta data most efficiently --- doc/source/algorithm.rst | 101 +++++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 1 + stream.py | 3 ++ 3 files changed, 105 insertions(+) create mode 100644 doc/source/algorithm.rst diff --git a/doc/source/algorithm.rst b/doc/source/algorithm.rst new file mode 100644 index 000000000..d1e4a9ba1 --- /dev/null +++ b/doc/source/algorithm.rst @@ -0,0 +1,101 @@ +######################## +Discussion of Algorithms +######################## + +************ +Introduction +************ +As you know, the pure-python object database support for GitPython is provided by the GitDB project. It is meant to be my backup plan to ensure that the DataVault (http://www.youtube.com/user/ByronBates99?feature=mhum#p/c/2A5C6EF5BDA8DB5C ) can handle reading huge files, especially those which were consolidated into packs. A nearly fully packed state is anticipated for the data-vaults repository, and reading these packs efficiently is an essential task. + +This document informs you about my findings in the struggle to improve the way packs are read to reduce memory usage required to handle huge files. It will try to conclude where future development could go to assure big delta-packed files can be read without the need of 8GB+ RAM. + +GitDB's main feature is the use of streams, hence the amount of memory used to read a database object is minimized, at the cost of some additional processing overhead to keep track of the stream state. This works great for legacy objects, which are essentially a zip-compressed byte-stream. + +Streaming data from delta-packed objects is far more difficult though, and only technically possible within certain limits, and at relatively high processing costs. My first observation was that there doesn't appear to be 'the one and only' algorithm which is generally superior. They all have their pros and cons, but fortunately this allows the implementation to choose the one most suited based on the amount of delta streams, as well as the size of the base, which allows an early and cheap estimate of the target size of the final data. + +********************************** +Traditional Delta-Apply-Algorithms +********************************** + +The brute-force CGit delta-apply algorithm +========================================== +CGit employs a simple and relatively brute-force algorithm, which resolves all delta streams recursively. When the recursion reaches the base level of the deltas, it will be decompressed into a buffer, then the first delta gets decompressed into a second buffer. From that, the target size of the delta can be extracted, to allocated a third buffer to hold the result of the operation, which consists of reading the delta stream byte-wise, to apply the operations in order, as described by single-byte opcodes. During recursion, each target buffer of the preceding delta-apply operation is used as base buffer for the next delta-apply operation, until the last delta was applied, leaving the final target buffer as result. + +About Delta-Opcodes +------------------- +There are only two kinds of opcodes, 'add-bytes' and 'copy-from-base'. One 'add-bytes' opcode can encode up to 7 bit of additional bytes to be copied from the delta stream into the target buffer. +A 'copy-from-base' opcode encodes a 32 bit offset into the base buffer, as well as the amount of bytes to be copied, which are up to 2^24 bytes. We may conclude that delta-bases may not be larger than 2^32+2^24 bytes in the current, extensible, implementation. +When generating the delta, git prefers copy operations over add operations, as they are much more efficient. Usually, the most recent, or biggest version of a file is used as base, whereas older and smaller versions of the file are expressed by copying only portions of the newest file. As it is not efficiently possible to represent all changes that way, add-bytes operations fill the gap where needed. All this explains why git can only add 128 bytes with one opcode, as it tries to minimize their use. +This implies that recent file history can usually be extracted faster than old history, which may involve many more deltas. + +Performance considerations +-------------------------- +The performance bottleneck of this algorithm appear to be the throughput of your RAM, as both opcodes will just trigger memcpy operations from one memory location to another, times the amount of deltas to apply. This in fact is very fast, even for big files above 100 MB. +Memory allocation could become an issue as you need the base buffer, the target buffer as well as the decompressed delta stream in memory at the same time. The continuous allocation and deallocation of possibly big buffers may support memory fragmentation. Whether it really kicks in, especially on 64 bit machines, is unproven though. +Nonetheless, the cgit implementation is currently the fastest one. + +The brute-force GitDB algorithm +=============================== +Despite of working essentially the same way as the CGit brute-force algorithm, GitDB minimizes the amount of allocations to 2 + num of deltas. The amount memory allocated peaks whiles the deltas are applied, as the base and target buffer, as well as the decompressed stream, are held in memory. +To achieve this, GitDB retrieves all delta-streams in advance, and peaks into their header information to determine the maximum size of the target buffer, just by reading 512 bytes of the compressed stream. If there is more than one delta to apply, the base buffer is set large enough to hold the biggest target buffer required by the delta streams. +Now it is possible to iterate all deltas, oldest first, newest last, and apply them using the buffers. At the end of each iteration, the buffers are swapped. + +Performance Results +------------------- +The performance test is performed on an aggressively packed repository with the history of cgit. 5000 sha's are extracted and read one after another. The delta-chains have a length of 0 to about 35. The pure-python implementation can stream the data of all objects (totaling 62,2 MiB) with an average rate of 8.1 MiB/s, which equals about 654 streams/s. +There are two bottlenecks: The major is the collection of the delta streams, which involves plenty of pack-lookup. This lookup is expensive in python, and is overly expensive. Its not overly critical though, as it only limits the amount of streams per second, not the actual data rate when applying the deltas. +Applying the deltas happens to be the second bottleneck, if the files to be processed get bigger. The more opcodes have to be processed, the more python slow function calls will dominate the result. As an example, it takes nearly 8 seconds to unpack a 125 MB file, where cgit only takes 2.4 s. + +To eliminate a few performance concerns, some key routines were rewritten in C. This changes the numbers of this particular test significantly, but not drastically, as the major bottleneck (delta collection) is still in place. Another performance reduction is due to the fact that plenty of other code related to the deltas is still pure-python. +Now all 5000 objects can be read at a rate of 11.1 MiB /s, or 892 streams/s. Fortunately, unpacking a big object is now done in 2.5s, which is just a tad slower than cgit, but with possibly less memory fragmentation. + + +************************************** +Paving the way towards delta streaming +************************************** + +GitDB's reverse delta aggregation algorithm +=========================================== +The idea of this algorithm is to merge all delta streams into one, which can then be applied in just one go. + +In the current implementation, delta streams are parsed into DeltaChunks (->**DC**), which are kept in vectors. Each DC represents one copy-from-base operation, or one or multiple consecutive add-bytes operations. DeltaChunks know about their target offset in the target buffer, and their size. Their target offsets are consecutive, i.e. one chunk ends where the next one begins, regarding their logical extend in the target buffer. +Add-bytes DCs additional store their data to apply, copy-from-base DCs store the offset into the base buffer from which to copy bytes. + +During processing, one starts with the latest (i.e. topmost) delta stream (->**TDS**), and iterates through its ancestor delta streams (->ADS) to merge them into the growing toplevel delta stream. + +The merging works by following a set of rules: + * Merge into the top-level delta from the youngest ancestor delta to the oldest one + * When merging one ADS, iterate from the first to the last chunk in TDS, then: + + * skip all add-bytes DCs. If bytes are added, these will always overwrite any operation coming from any ADS at the same offset. + * copy-from-base DCs will copy a slice of the respective portion of the ADS ( as defined by their base offset ) and use it to replace the original chunk. This acts as a 'virtual' copy-from-base operation. + + * Finish the merge once all ADS have been handled, or once the TDS only consists of add-byte DCs. The remaining copy-from-base DCs will copy from the original base buffer accordingly. + +Applying the TDS is as straightforward as applying any other DS. The base buffer is required to be kept in memory. In the current implementation, a full-size target buffer is allocated to hold the result of applying the chunk information. + +The memory consumption during the TDS processing are the uncompressed delta-bytes, the parsed DS, as well as the TDS. Afterwards one requires an allocated base buffer, the target buffer, as well as the TDS. +It is clearly visible that the current implementation does not at all reduce memory consumption, but the opposite is true as the TDS can be large for large files. + +Performance Results +------------------- +The benchmarking context was the same as for the brute-force GitDB algorithm. This implementation is far more complex than the said brute-force implementation, which clearly reflects in the numbers. It's pure-python throughput is at only 1.1 MiB/s, which equals 89 streams/s. +The biggest performance bottleneck is the slicing of the parsed delta streams, where the program spends most of its time due to hundred thousands of calls. + +To get a more usable version of the algorithm, it was implemented in C, such that python must do no more than two calls to get all the work done. The first prepares the TDS, the second applies it, writing it into a target buffer. +The throughput reaches 15.6 MiB/s, which equals 1267 streams/s, which makes it 14 times faster than the pure python version, and amazingly even 1.4 times faster than the brute-force C implementation. + +*TODO* +All this comes at a relatively high memory consumption, and heavily degrading performance with raising file sizes. A 125 MB file took 8 seconds to unpack for instance. The reason for this is the current implementation's brute-force algorithm to insert ADS slices into the TDS, which triggers an enormous amount of memmove operations of large portions of overlapping memory portions. + +Additionally, with each new level being merged, not only are more DCs inserted, but the new chunks may get smaller as well. This can reach a point where one chunk only represents an individual byte, so the size of the data structure outweighs the logical chunk size by far. + + +Future work +=========== +* Analyse TDS to determine which sections from base buffer need to be copied. Read these in order of lowest to highest offset from base stream, and copy them into a smaller memory map. Relink source offsets to point to the new location in the new buffer. +* recompress TDS into bytestream to minimize the memory footprint when streaming, allocate the stream into a memory map. + +With that in place, streaming becomes trivial. Memory consumption + + diff --git a/doc/source/index.rst b/doc/source/index.rst index d414cb22f..5223e6bd9 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -14,6 +14,7 @@ Contents: intro tutorial api + algorithm changes Indices and tables diff --git a/stream.py b/stream.py index f5e05e114..38c86dae7 100644 --- a/stream.py +++ b/stream.py @@ -379,6 +379,9 @@ def _set_cache_(self, attr): def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" + # TODO: There should be a special case if there is only one stream + # Then the default-git algorithm should perform a tad faster, as the + # delta is not peaked into, causing less overhead. buffer_info_list = list() max_target_size = 0 for dstream in self._dstreams: From 6656c66f7edcf746132ce52d9ed714283140eaa0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 20:39:20 +0200 Subject: [PATCH 0101/1392] Implemented simple pre-pass to count offsets to help calculate where each chunk is going to be. This way, memmove becomes memcpy, and only one grow is required --- _delta_apply.c | 189 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 129 insertions(+), 60 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index dfb0a94c3..76d482b0d 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -11,7 +11,7 @@ typedef unsigned char uchar; typedef uchar bool; // Constants -const ull gDVC_grow_by = 100; +const ull gDCV_grow_by = 100; #ifdef DEBUG #define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) @@ -170,8 +170,8 @@ int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) return 1; } - if (num_dc - vec->reserved_size){ - num_dc += gDVC_grow_by; + if (num_dc - vec->reserved_size < 10){ + num_dc += gDCV_grow_by; } #ifdef DEBUG @@ -255,6 +255,12 @@ ull DCV_rbound(const DeltaChunkVector* vec) return DC_rbound(DCV_last(vec)); } +inline +ull DCV_size(const DeltaChunkVector* vec) +{ + return DCV_rbound(vec) - DCV_lbound(vec); +} + inline int DCV_empty(const DeltaChunkVector* vec) { @@ -269,6 +275,14 @@ const DeltaChunk* DCV_end(const DeltaChunkVector* vec) return vec->mem + vec->size; } +// return first item in vector +inline +DeltaChunk* DCV_first(const DeltaChunkVector* vec) +{ + assert(!DCV_empty(vec)); + return vec->mem; +} + void DCV_destroy(DeltaChunkVector* vec) { if (vec->mem){ @@ -306,7 +320,7 @@ void DCV_reset(DeltaChunkVector* vec) if (vec->size == 0) return; - DeltaChunk* dc = vec->mem; + DeltaChunk* dc = DCV_first(vec); const DeltaChunk* dcend = DCV_end(vec); for(;dc < dcend; dc++){ DC_destroy(dc); @@ -322,7 +336,7 @@ static inline DeltaChunk* DCV_append(DeltaChunkVector* vec) { if (vec->size + 1 > vec->reserved_size){ - DCV_grow_by(vec, gDVC_grow_by); + DCV_grow_by(vec, gDCV_grow_by); } DeltaChunk* next = vec->mem + vec->size; @@ -364,7 +378,7 @@ int DCV_dbg_check_integrity(const DeltaChunkVector* vec) if(DCV_empty(vec)){ return 0; } - const DeltaChunk* i = vec->mem; + const DeltaChunk* i = DCV_first(vec); const DeltaChunk* end = DCV_end(vec); ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); @@ -380,7 +394,7 @@ int DCV_dbg_check_integrity(const DeltaChunkVector* vec) } const DeltaChunk* endm1 = DCV_end(vec) - 1; - for(i = vec->mem; i < endm1; i++){ + for(i = DCV_first(vec); i < endm1; i++){ const DeltaChunk* n = i+1; if (DC_rbound(i) != n->to){ return 0; @@ -390,46 +404,82 @@ int DCV_dbg_check_integrity(const DeltaChunkVector* vec) return 1; } +// Return the amount of chunks a slice at the given spot would have +inline +uint DCV_count_slice_chunks(const DeltaChunkVector* src, ull ofs, ull size) +{ + uint num_dc = 0; + DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + + // partial overlap + if (cdc->to != ofs) { + const ull relofs = ofs - cdc->to; + size -= cdc->ts - relofs < size ? cdc->ts - relofs : size; + num_dc += 1; + cdc += 1; + + if (size == 0){ + return num_dc; + } + } + + const DeltaChunk* vecend = DCV_end(src); + for( ;(cdc < vecend) && size; ++cdc){ + num_dc += 1; + if (cdc->ts < size) { + size -= cdc->ts; + } else { + size = 0; + break; + } + } + + return num_dc; +} + // Write a slice as defined by its absolute offset in bytes and its size into the given -// destination. The individual chunks written will be a deep copy of the source +// destination memory. The individual chunks written will be a deep copy of the source // data chunks -// TODO: this could trigger copying many smallish add-chunk pieces - maybe some sort -// of append-only memory pool would improve performance +// Return: number of chunks in the slice inline -void DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunkVector* dest, ull ofs, ull size) +uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, ull size) { assert(DCV_lbound(src) <= ofs); assert((ofs + size) <= DCV_rbound(src)); DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + uint num_chunks = 0; // partial overlap if (cdc->to != ofs) { - DeltaChunk* destc = DCV_append(dest); const ull relofs = ofs - cdc->to; - DC_offset_copy_to(cdc, destc, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); + DC_offset_copy_to(cdc, dest, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); cdc += 1; - size -= destc->ts; + size -= dest->ts; + dest += 1; + num_chunks += 1; if (size == 0){ - return; + return num_chunks; } } const DeltaChunk* vecend = DCV_end(src); for( ;(cdc < vecend) && size; ++cdc) { + num_chunks += 1; if (cdc->ts < size) { - DC_copy_to(cdc, DCV_append(dest)); + DC_copy_to(cdc, dest++); size -= cdc->ts; } else { - DC_offset_copy_to(cdc, DCV_append(dest), 0, size); + DC_offset_copy_to(cdc, dest++, 0, size); size = 0; break; } } assert(size == 0); + return num_chunks; } @@ -458,64 +508,85 @@ void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, } // Finally copy all the items in - memcpy((void*) at, (void*)from->mem, from->size*sizeof(DeltaChunk)); + memcpy((void*) at, (void*)DCV_first(from), from->size*sizeof(DeltaChunk)); // FINALLY: update size to->size += from->size - 1; } // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost -// delta to apply. tmpl is used as temporary space and must be initialzed and destroyed by the -// caller -void DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv, DeltaChunkVector* tmpl) +// delta to apply. +bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) { - Py_ssize_t dci = 0; - Py_ssize_t iend = tdcv->size; - DeltaChunk* dc; - DBG_check(tdcv); DBG_check(bdcv); - for (;dci < iend; dci++) + uint* offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); + if (!offset_array){ + return 0; + } + + fprintf(stderr, "old size = %i\n", (int)tdcv->size); + uint* pofs = offset_array; + uint num_addchunks = 0; + + DeltaChunk* dc = DCV_first(tdcv); + const DeltaChunk* dcend = DCV_end(tdcv); + const ull oldsize = DCV_size(tdcv); + + // OFFSET RUN + for (;dc < dcend; dc++, pofs++) { // Data chunks don't need processing - dc = DCV_get(tdcv, dci); + *pofs = num_addchunks; if (dc->data){ continue; } - // Copy Chunk Handling - DCV_copy_slice_to(bdcv, tmpl, dc->so, dc->ts); - DBG_check(tmpl); - assert(tmpl->size); - - // move target bounds - DeltaChunk* tdc = tmpl->mem; - DeltaChunk* tdcend = tmpl->mem + tmpl->size; - const ull ofs = dc->to - dc->so; - for(;tdc < tdcend; tdc++){ - tdc->to += ofs; + // offset the next chunk by the amount of chunks in the slice + // - 1, because we replace our own chunk + num_addchunks += DCV_count_slice_chunks(bdcv, dc->so, dc->ts) - 1; + } + + // reserve enough memory to hold all the new chunks + // reinit pointers, array could have been reallocated + DCV_reserve_memory(tdcv, tdcv->size + num_addchunks); + dc = DCV_last(tdcv); + dcend = DCV_first(tdcv) - 1; + + // now, that we have our pointers with the old size + tdcv->size += num_addchunks; + + // Insert slices, from the end to the beginning, which allows memcpy + // to be used, with a little help of the offset array + for (pofs -= 1; dc > dcend; dc--, pofs-- ) + { + // Data chunks don't need processing + const uint ofs = *pofs; + if (dc->data){ + // TODO: peak the preceeding chunks to figure out whether they are + // all just moved by ofs. In that case, they can move as a whole! + // just copy the chunk according to its offset + if (ofs){ + memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaChunk)); + } + continue; } - // insert slice into our list - if (tmpl->size == 1){ - // Its not data, so destroy is not really required, anyhow ... - DC_destroy(dc); - *dc = *DCV_get(tmpl, 0); - } else { - DCV_reserve_memory(tdcv, tdcv->size + tmpl->size - 1 + gDVC_grow_by); - dc = DCV_get(tdcv, dci); - DCV_replace_one_by_many(tmpl, tdcv, dc); - // Compensate for us being replaced - dci += tmpl->size-1; - iend += tmpl->size-1; + // Copy Chunks, and move their target offset into place + DeltaChunk* tdc = dc + ofs; + DeltaChunk* tdcend = tdc + DCV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); + const ull relofs = dc->to - dc->so; + for(;tdc < tdcend; tdc++){ + tdc->to += relofs; } - - DBG_check(tdcv); - - // make sure the members will not be deallocated by the list - DCV_forget_members(tmpl); } + + fprintf(stderr, "NEW size = %i\n", (int)tdcv->size); + DBG_check(tdcv); + assert(DCV_size(tdcv) == oldsize); + PyMem_Free(offset_array); + return 1; } // DELTA CHUNK LIST (PYTHON) @@ -699,10 +770,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DeltaChunkVector dcv; DeltaChunkVector tdcv; - DeltaChunkVector tmpl; DCV_init(&dcv, 100); // should be enough to keep the average text file DCV_init(&tdcv, 0); - DCV_init(&tmpl, 200); unsigned int dsi = 0; PyObject* ds = 0; @@ -725,7 +794,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const ull base_size = msb_size(&data, dend); const ull target_size = msb_size(&data, dend); - // estimate number of ops - assume one third adds, half two byte (size+offset) copies // Assume good compression for the adds const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); DCV_reserve_memory(&dcv, approx_num_cmds); @@ -824,7 +892,9 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } if (!is_first_run){ - DCV_connect_with_base(&tdcv, &dcv, &tmpl); + if (!DCV_connect_with_base(&tdcv, &dcv)){ + error = 1; + } } #ifdef DEBUG @@ -859,7 +929,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(stream_iter); } - DCV_destroy(&tmpl); if (dsi > 1){ // otherwise dcv equals tcl DCV_destroy(&dcv); From c03a46bea58d9b108cb314f9a1f0c422c05bb3bf Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 21:07:21 +0200 Subject: [PATCH 0102/1392] Fixed tiny little bug that would cause our own chunk to be overridden before we make one last computation with its unaltered values --- _delta_apply.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 76d482b0d..4cdb654f2 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -456,7 +456,7 @@ uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, u DC_offset_copy_to(cdc, dest, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); cdc += 1; size -= dest->ts; - dest += 1; + dest += 1; // must be here, we are reading the size ! num_chunks += 1; if (size == 0){ @@ -472,7 +472,7 @@ uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, u DC_copy_to(cdc, dest++); size -= cdc->ts; } else { - DC_offset_copy_to(cdc, dest++, 0, size); + DC_offset_copy_to(cdc, dest, 0, size); size = 0; break; } @@ -526,7 +526,6 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) return 0; } - fprintf(stderr, "old size = %i\n", (int)tdcv->size); uint* pofs = offset_array; uint num_addchunks = 0; @@ -574,17 +573,19 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) } // Copy Chunks, and move their target offset into place + // As we could override dc when slicing, we get the data here + const ull relofs = dc->to - dc->so; + DeltaChunk* tdc = dc + ofs; DeltaChunk* tdcend = tdc + DCV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); - const ull relofs = dc->to - dc->so; for(;tdc < tdcend; tdc++){ tdc->to += relofs; } } - fprintf(stderr, "NEW size = %i\n", (int)tdcv->size); DBG_check(tdcv); assert(DCV_size(tdcv) == oldsize); + PyMem_Free(offset_array); return 1; } From 65c9abfde0eae317c6c6dcf91918258e5e6ca33c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 21:38:48 +0200 Subject: [PATCH 0103/1392] removed some debug code --- _delta_apply.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 4cdb654f2..8cfb3f2e8 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -521,7 +521,7 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) DBG_check(tdcv); DBG_check(bdcv); - uint* offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); + uint *const offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); if (!offset_array){ return 0; } @@ -531,7 +531,6 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) DeltaChunk* dc = DCV_first(tdcv); const DeltaChunk* dcend = DCV_end(tdcv); - const ull oldsize = DCV_size(tdcv); // OFFSET RUN for (;dc < dcend; dc++, pofs++) @@ -563,9 +562,10 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) // Data chunks don't need processing const uint ofs = *pofs; if (dc->data){ - // TODO: peak the preceeding chunks to figure out whether they are + // NOTE: could peek the preceeding chunks to figure out whether they are // all just moved by ofs. In that case, they can move as a whole! - // just copy the chunk according to its offset + // tests showed that this is very rare though, even in huge deltas, so its + // not worth the extra effort if (ofs){ memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaChunk)); } @@ -584,7 +584,6 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) } DBG_check(tdcv); - assert(DCV_size(tdcv) == oldsize); PyMem_Free(offset_array); return 1; From 78665b13ff4125f4ce3e5311d040c027bdc92a9a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 14 Oct 2010 22:59:19 +0200 Subject: [PATCH 0104/1392] Updated draft with latest data, finished it by defining future ways to improve the algorithm --- doc/source/algorithm.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/doc/source/algorithm.rst b/doc/source/algorithm.rst index d1e4a9ba1..55207b67b 100644 --- a/doc/source/algorithm.rst +++ b/doc/source/algorithm.rst @@ -83,19 +83,17 @@ The benchmarking context was the same as for the brute-force GitDB algorithm. Th The biggest performance bottleneck is the slicing of the parsed delta streams, where the program spends most of its time due to hundred thousands of calls. To get a more usable version of the algorithm, it was implemented in C, such that python must do no more than two calls to get all the work done. The first prepares the TDS, the second applies it, writing it into a target buffer. -The throughput reaches 15.6 MiB/s, which equals 1267 streams/s, which makes it 14 times faster than the pure python version, and amazingly even 1.4 times faster than the brute-force C implementation. +The throughput reaches 16.7 MiB/s, which equals 1344 streams/s, which makes it 15 times faster than the pure python version, and amazingly even 1.5 times faster than the brute-force C implementation. As a comparison, cgit is able to stream about 20 MiB when controlling it through a pipe. GitDBs performance may still improve once pack access is reimplemented in C as well. -*TODO* -All this comes at a relatively high memory consumption, and heavily degrading performance with raising file sizes. A 125 MB file took 8 seconds to unpack for instance. The reason for this is the current implementation's brute-force algorithm to insert ADS slices into the TDS, which triggers an enormous amount of memmove operations of large portions of overlapping memory portions. +All this comes at a relatively high memory consumption.Additionally, with each new level being merged, not only are more DCs inserted, but the new chunks may get smaller as well. This can reach a point where one chunk only represents an individual byte, so the size of the data structure outweighs the logical chunk size by far. -Additionally, with each new level being merged, not only are more DCs inserted, but the new chunks may get smaller as well. This can reach a point where one chunk only represents an individual byte, so the size of the data structure outweighs the logical chunk size by far. +A 125 MB file took 3.1 seconds to unpack for instance, which is only 33% slower than the c implementation of the brute-force algorithm. Future work =========== -* Analyse TDS to determine which sections from base buffer need to be copied. Read these in order of lowest to highest offset from base stream, and copy them into a smaller memory map. Relink source offsets to point to the new location in the new buffer. -* recompress TDS into bytestream to minimize the memory footprint when streaming, allocate the stream into a memory map. - -With that in place, streaming becomes trivial. Memory consumption +The current implementation of the reverse delta aggregation algorithm is already working well and fast, but leaves room for improvement in the realm of its memory consumption. One way to considerably reduce it would be to index the delta stream to determine bounds, instead of parsing it into a separate data structure +Another very promising option is that streaming of delta data is indeed possible. Depending on the configuration of the copy-from-base operations, different optimizations could be applied to reduce the amount of memory required for the final processed delta stream. Some configurations may even allow it to stream data from the base buffer, instead of pre-loading it for random access. +The ability to stream files at reduced memory costs would only be feasible for big files, and would have to be payed with extra pre-processing time. From ffafb2e998220344f4674b25b6760254b2ec453e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Oct 2010 16:04:17 +0200 Subject: [PATCH 0105/1392] First adjustment to prepare the algorithm to work on deltastreams directly, without an intermediate conversion into far-too-large DeltaChunks. The new style just uses a single index, using one-fourth of the memory --- _delta_apply.c | 329 ++++++++++++++----------------------------------- 1 file changed, 93 insertions(+), 236 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 8cfb3f2e8..9768b5264 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -11,82 +11,38 @@ typedef unsigned char uchar; typedef uchar bool; // Constants -const ull gDCV_grow_by = 100; +const ull gDIV_grow_by = 100; + + +// DELTA INFO +///////////// +typedef struct { + uint dso; // delta stream offset + uint to; // target offset (cache) +} DeltaInfo; -#ifdef DEBUG -#define DBG_check(vec) assert(DCV_dbg_check_integrity(vec)) -#else -#define DBG_check(vec) -#endif // DELTA CHUNK //////////////// // Internal Delta Chunk Objects +// They are just used to keep information parsed from a stream +// The data pointer is always shared typedef struct { ull to; ull ts; ull so; const uchar* data; - bool data_shared; } DeltaChunk; inline -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so) +void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, const uchar* data) { dc->to = to; dc->ts = ts; dc->so = so; dc->data = NULL; - dc->data_shared = 0; -} - -inline -void DC_deallocate_data(DeltaChunk* dc) -{ - if (!dc->data_shared && dc->data){ - PyMem_Free((void*)dc->data); - } - dc->data = NULL; } -inline -void DC_destroy(DeltaChunk* dc) -{ - DC_deallocate_data(dc); -} - -// Store a copy of data in our instance. If shared is 1, the data will be shared, -// hence it will only be stored, but the memory will not be touched, or copied. -inline -void DC_set_data(DeltaChunk* dc, const uchar* data, Py_ssize_t dlen, bool shared) -{ - DC_deallocate_data(dc); - - if (data == 0){ - dc->data = NULL; - dc->data_shared = 0; - return; - } - - dc->data_shared = shared; - if (shared){ - dc->data = data; - } else { - dc->data = (uchar*)PyMem_Malloc(dlen); - memcpy((void*)dc->data, (void*)data, dlen); - } - -} - -// Make the given data our own. It is assumed to have the size stored in our instance -// and will be managed by us. -inline -void DC_set_data_with_ownership(DeltaChunk* dc, const uchar* data) -{ - assert(data); - DC_deallocate_data(dc); - dc->data = data; -} inline ull DC_rbound(const DeltaChunk* dc) @@ -94,7 +50,8 @@ ull DC_rbound(const DeltaChunk* dc) return dc->to + dc->ts; } -// Apply +// Apply +// TODO: remove, just left it for reference inline void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) { @@ -114,48 +71,16 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec } -// Copy all data from src to dest, the data pointer will be copied too -inline -void DC_copy_to(const DeltaChunk* src, DeltaChunk* dest) -{ - dest->to = src->to; - dest->ts = src->ts; - dest->so = src->so; - dest->data_shared = 0; - dest->data = NULL; - - DC_set_data(dest, src->data, src->ts, 0); -} - -// Copy all data with the given offset and size. The source offset, as well -// as the data will be truncated accordingly -inline -void DC_offset_copy_to(const DeltaChunk* src, DeltaChunk* dest, ull ofs, ull size) -{ - assert(size <= src->ts); - assert(src->to + ofs + size <= DC_rbound(src)); - - dest->to = src->to + ofs; - dest->ts = size; - dest->so = src->so + ofs; - dest->data = NULL; - - if (src->data){ - DC_set_data(dest, src->data + ofs, size, 0); - } else { - dest->data_shared = 0; - } -} - // DELTA CHUNK VECTOR ///////////////////// typedef struct { - DeltaChunk* mem; // Memory - Py_ssize_t size; // Size in DeltaChunks - Py_ssize_t reserved_size; // Reserve in DeltaChunks -} DeltaChunkVector; + DeltaInfo* mem; // Memory + const uchar* dstream; // pointer to delta stream we index + Py_ssize_t size; // Size in DeltaInfos + Py_ssize_t reserved_size; // Reserve in DeltaInfos +} DeltaInfoVector; @@ -164,14 +89,14 @@ typedef struct { // NOTE: added a minimum allocation to assure reallocation is not done // just for a single additional entry. DCVs change often, and reallocs are expensive inline -int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) +int DIV_reserve_memory(DeltaInfoVector* vec, uint num_dc) { if (num_dc <= vec->reserved_size){ return 1; } if (num_dc - vec->reserved_size < 10){ - num_dc += gDCV_grow_by; + num_dc += gDIV_grow_by; } #ifdef DEBUG @@ -179,9 +104,9 @@ int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) #endif if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaChunk)); + vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaInfo)); } else { - vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaChunk)); + vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaInfo)); } if (vec->mem == NULL){ @@ -194,7 +119,7 @@ int DCV_reserve_memory(DeltaChunkVector* vec, uint num_dc) const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; if (!was_null) format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; - fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaChunk)), vec->mem, (int)vec->reserved_size); + fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaInfo)), vec->mem, (int)vec->reserved_size); #endif return vec->mem != NULL; @@ -207,28 +132,28 @@ large enough. Return 1 on success, 0 on failure */ inline -int DCV_grow_by(DeltaChunkVector* vec, uint num_dc) +int DIV_grow_by(DeltaInfoVector* vec, uint num_dc) { - return DCV_reserve_memory(vec, vec->reserved_size + num_dc); + return DIV_reserve_memory(vec, vec->reserved_size + num_dc); } -int DCV_init(DeltaChunkVector* vec, ull initial_size) +int DIV_init(DeltaInfoVector* vec, ull initial_size) { vec->mem = NULL; vec->size = 0; vec->reserved_size = 0; - return DCV_grow_by(vec, initial_size); + return DIV_grow_by(vec, initial_size); } inline -ull DCV_len(const DeltaChunkVector* vec) +ull DIV_len(const DeltaInfoVector* vec) { return vec->size; } inline -ull DCV_lbound(const DeltaChunkVector* vec) +ull DIV_lbound(const DeltaInfoVector* vec) { assert(vec->size && vec->mem); return vec->mem->to; @@ -236,7 +161,7 @@ ull DCV_lbound(const DeltaChunkVector* vec) // Return item at index inline -DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) +DeltaInfo* DIV_get(const DeltaInfoVector* vec, Py_ssize_t i) { assert(i < vec->size && vec->mem); return &vec->mem[i]; @@ -244,54 +169,54 @@ DeltaChunk* DCV_get(const DeltaChunkVector* vec, Py_ssize_t i) // Return last item inline -DeltaChunk* DCV_last(const DeltaChunkVector* vec) +DeltaInfo* DIV_last(const DeltaInfoVector* vec) { - return DCV_get(vec, vec->size-1); + return DIV_get(vec, vec->size-1); } inline -ull DCV_rbound(const DeltaChunkVector* vec) +ull DIV_rbound(const DeltaInfoVector* vec) { - return DC_rbound(DCV_last(vec)); + return DC_rbound(DIV_last(vec)); } inline -ull DCV_size(const DeltaChunkVector* vec) +ull DIV_size(const DeltaInfoVector* vec) { - return DCV_rbound(vec) - DCV_lbound(vec); + return DIV_rbound(vec) - DIV_lbound(vec); } inline -int DCV_empty(const DeltaChunkVector* vec) +int DIV_empty(const DeltaInfoVector* vec) { return vec->size == 0; } // Return end pointer of the vector inline -const DeltaChunk* DCV_end(const DeltaChunkVector* vec) +const DeltaInfo* DIV_end(const DeltaInfoVector* vec) { - assert(!DCV_empty(vec)); + assert(!DIV_empty(vec)); return vec->mem + vec->size; } // return first item in vector inline -DeltaChunk* DCV_first(const DeltaChunkVector* vec) +DeltaInfo* DIV_first(const DeltaInfoVector* vec) { - assert(!DCV_empty(vec)); + assert(!DIV_empty(vec)); return vec->mem; } -void DCV_destroy(DeltaChunkVector* vec) +void DIV_destroy(DeltaInfoVector* vec) { if (vec->mem){ #ifdef DEBUG fprintf(stderr, "Freeing %p\n", (void*)vec->mem); #endif - const DeltaChunk* end = &vec->mem[vec->size]; - DeltaChunk* i; + const DeltaInfo* end = &vec->mem[vec->size]; + DeltaInfo* i; for(i = vec->mem; i < end; i++){ DC_destroy(i); } @@ -306,7 +231,7 @@ void DCV_destroy(DeltaChunkVector* vec) // Reset this vector so that its existing memory can be filled again. // Memory will be kept, but not cleaned up inline -void DCV_forget_members(DeltaChunkVector* vec) +void DIV_forget_members(DeltaInfoVector* vec) { vec->size = 0; } @@ -315,13 +240,13 @@ void DCV_forget_members(DeltaChunkVector* vec) // have been deallocated properly. // It will keep its memory though, and hence can be filled again inline -void DCV_reset(DeltaChunkVector* vec) +void DIV_reset(DeltaInfoVector* vec) { if (vec->size == 0) return; - DeltaChunk* dc = DCV_first(vec); - const DeltaChunk* dcend = DCV_end(vec); + DeltaInfo* dc = DIV_first(vec); + const DeltaInfo* dcend = DIV_end(vec); for(;dc < dcend; dc++){ DC_destroy(dc); } @@ -333,27 +258,27 @@ void DCV_reset(DeltaChunkVector* vec) // Append one chunk to the end of the list, and return a pointer to it // It will not have been initialized ! static inline -DeltaChunk* DCV_append(DeltaChunkVector* vec) +DeltaInfo* DIV_append(DeltaInfoVector* vec) { if (vec->size + 1 > vec->reserved_size){ - DCV_grow_by(vec, gDCV_grow_by); + DIV_grow_by(vec, gDIV_grow_by); } - DeltaChunk* next = vec->mem + vec->size; + DeltaInfo* next = vec->mem + vec->size; vec->size += 1; return next; } // Return delta chunk being closest to the given absolute offset inline -DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) +DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) { assert(vec->mem); ull lo = 0; ull hi = vec->size; ull mid; - DeltaChunk* dc; + DeltaInfo* dc; while (lo < hi) { @@ -368,48 +293,16 @@ DeltaChunk* DCV_closest_chunk(const DeltaChunkVector* vec, ull ofs) } } - return DCV_last(vec); + return DIV_last(vec); } -// Assert the given vector has correct datachunks -// return 1 on success -int DCV_dbg_check_integrity(const DeltaChunkVector* vec) -{ - if(DCV_empty(vec)){ - return 0; - } - const DeltaChunk* i = DCV_first(vec); - const DeltaChunk* end = DCV_end(vec); - - ull aparent_size = DCV_rbound(vec) - DCV_lbound(vec); - ull acc_size = 0; - for(; i < end; i++){ - acc_size += i->ts; - } - if (acc_size != aparent_size) - return 0; - - if (vec->size < 2){ - return 1; - } - - const DeltaChunk* endm1 = DCV_end(vec) - 1; - for(i = DCV_first(vec); i < endm1; i++){ - const DeltaChunk* n = i+1; - if (DC_rbound(i) != n->to){ - return 0; - } - } - - return 1; -} // Return the amount of chunks a slice at the given spot would have inline -uint DCV_count_slice_chunks(const DeltaChunkVector* src, ull ofs, ull size) +uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) { uint num_dc = 0; - DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + DeltaInfo* cdc = DIV_closest_chunk(src, ofs); // partial overlap if (cdc->to != ofs) { @@ -423,7 +316,7 @@ uint DCV_count_slice_chunks(const DeltaChunkVector* src, ull ofs, ull size) } } - const DeltaChunk* vecend = DCV_end(src); + const DeltaInfo* vecend = DIV_end(src); for( ;(cdc < vecend) && size; ++cdc){ num_dc += 1; if (cdc->ts < size) { @@ -442,12 +335,12 @@ uint DCV_count_slice_chunks(const DeltaChunkVector* src, ull ofs, ull size) // data chunks // Return: number of chunks in the slice inline -uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, ull size) +uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull size) { - assert(DCV_lbound(src) <= ofs); - assert((ofs + size) <= DCV_rbound(src)); + assert(DIV_lbound(src) <= ofs); + assert((ofs + size) <= DIV_rbound(src)); - DeltaChunk* cdc = DCV_closest_chunk(src, ofs); + DeltaInfo* cdc = DIV_closest_chunk(src, ofs); uint num_chunks = 0; // partial overlap @@ -464,7 +357,7 @@ uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, u } } - const DeltaChunk* vecend = DCV_end(src); + const DeltaInfo* vecend = DIV_end(src); for( ;(cdc < vecend) && size; ++cdc) { num_chunks += 1; @@ -483,44 +376,10 @@ uint DCV_copy_slice_to(const DeltaChunkVector* src, DeltaChunk* dest, ull ofs, u } -// Insert all chunks in 'from' to 'to', starting at the delta chunk named 'at' which -// originates in to -// 'at' will be replaced by the items to insert ( special purpose ) -// 'at' will be properly destroyed, but all items will just be copied bytewise -// using memcpy. Hence from must just forget about them ! -// IMPORTANT: to must have an appropriate size already -inline -void DCV_replace_one_by_many(const DeltaChunkVector* from, DeltaChunkVector* to, DeltaChunk* at) -{ - assert(from->size > 1); - assert(to->size + from->size - 1 <= to->reserved_size); - - // -1 because we replace 'at' - DC_destroy(at); - - // If we are somewhere in the middle, we have to make some space - if (DCV_last(to) != at) { - // IMPORTANT: This memmove kills the performance in case of large deltas - // Causing everything to slow down enormously. Its logical, as the memory - // gets shifted each time we insert nodes, for each chunk, for ever smaller - // chunks depending on the deltas - memmove((void*)(at+from->size), (void*)(at+1), (size_t)((DCV_end(to) - (at+1)) * sizeof(DeltaChunk))); - } - - // Finally copy all the items in - memcpy((void*) at, (void*)DCV_first(from), from->size*sizeof(DeltaChunk)); - - // FINALLY: update size - to->size += from->size - 1; -} - // Take slices of bdcv into the corresponding area of the tdcv, which is the topmost // delta to apply. -bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) +bool DIV_connect_with_base(DeltaInfoVector* tdcv, const DeltaInfoVector* bdcv) { - DBG_check(tdcv); - DBG_check(bdcv); - uint *const offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); if (!offset_array){ return 0; @@ -529,8 +388,8 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) uint* pofs = offset_array; uint num_addchunks = 0; - DeltaChunk* dc = DCV_first(tdcv); - const DeltaChunk* dcend = DCV_end(tdcv); + DeltaInfo* dc = DIV_first(tdcv); + const DeltaInfo* dcend = DIV_end(tdcv); // OFFSET RUN for (;dc < dcend; dc++, pofs++) @@ -543,14 +402,14 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) // offset the next chunk by the amount of chunks in the slice // - 1, because we replace our own chunk - num_addchunks += DCV_count_slice_chunks(bdcv, dc->so, dc->ts) - 1; + num_addchunks += DIV_count_slice_chunks(bdcv, dc->so, dc->ts) - 1; } // reserve enough memory to hold all the new chunks // reinit pointers, array could have been reallocated - DCV_reserve_memory(tdcv, tdcv->size + num_addchunks); - dc = DCV_last(tdcv); - dcend = DCV_first(tdcv) - 1; + DIV_reserve_memory(tdcv, tdcv->size + num_addchunks); + dc = DIV_last(tdcv); + dcend = DIV_first(tdcv) - 1; // now, that we have our pointers with the old size tdcv->size += num_addchunks; @@ -567,7 +426,7 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) // tests showed that this is very rare though, even in huge deltas, so its // not worth the extra effort if (ofs){ - memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaChunk)); + memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaInfo)); } continue; } @@ -576,26 +435,24 @@ bool DCV_connect_with_base(DeltaChunkVector* tdcv, const DeltaChunkVector* bdcv) // As we could override dc when slicing, we get the data here const ull relofs = dc->to - dc->so; - DeltaChunk* tdc = dc + ofs; - DeltaChunk* tdcend = tdc + DCV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); + DeltaInfo* tdc = dc + ofs; + DeltaInfo* tdcend = tdc + DIV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); for(;tdc < tdcend; tdc++){ tdc->to += relofs; } } - DBG_check(tdcv); - PyMem_Free(offset_array); return 1; } // DELTA CHUNK LIST (PYTHON) ///////////////////////////// - +// Internally, it has nothing to do with a ChunkList anymore though typedef struct { PyObject_HEAD // ----------- - DeltaChunkVector vec; + DeltaInfoVector vec; } DeltaChunkList; @@ -608,28 +465,28 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) return -1; } - DCV_init(&self->vec, 0); + DIV_init(&self->vec, 0); return 0; } static void DCL_dealloc(DeltaChunkList* self) { - DCV_destroy(&(self->vec)); + DIV_destroy(&(self->vec)); } static PyObject* DCL_len(DeltaChunkList* self) { - return PyLong_FromUnsignedLongLong(DCV_len(&self->vec)); + return PyLong_FromUnsignedLongLong(DIV_len(&self->vec)); } static inline ull DCL_rbound(DeltaChunkList* self) { - if (DCV_empty(&self->vec)) + if (DIV_empty(&self->vec)) return 0; - return DCV_rbound(&self->vec); + return DIV_rbound(&self->vec); } static @@ -660,7 +517,7 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) } const DeltaChunk* i = self->vec.mem; - const DeltaChunk* end = DCV_end(&self->vec); + const DeltaChunk* end = DIV_end(&self->vec); const uchar* data; Py_ssize_t dlen; @@ -768,10 +625,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) stream_iter = dstreams; } - DeltaChunkVector dcv; - DeltaChunkVector tdcv; - DCV_init(&dcv, 100); // should be enough to keep the average text file - DCV_init(&tdcv, 0); + DeltaInfoVector dcv; + DeltaInfoVector tdcv; + DIV_init(&dcv, 100); // should be enough to keep the average text file + DIV_init(&tdcv, 0); unsigned int dsi = 0; PyObject* ds = 0; @@ -796,7 +653,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // Assume good compression for the adds const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); - DCV_reserve_memory(&dcv, approx_num_cmds); + DIV_reserve_memory(&dcv, approx_num_cmds); // parse command stream ull tbw = 0; // Amount of target bytes written @@ -829,7 +686,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) break; } - DC_init(DCV_append(&dcv), tbw, cp_size, cp_off); + DC_init(DIV_append(&dcv), tbw, cp_size, cp_off); tbw += cp_size; } else if (cmd) { @@ -862,7 +719,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } #endif - DeltaChunk* dc = DCV_append(&dcv); + DeltaChunk* dc = DIV_append(&dcv); DC_init(dc, tbw, num_bytes, 0); // gather the data, or (possibly) share single blocks @@ -892,7 +749,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } if (!is_first_run){ - if (!DCV_connect_with_base(&tdcv, &dcv)){ + if (!DIV_connect_with_base(&tdcv, &dcv)){ error = 1; } } @@ -905,10 +762,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (is_first_run){ tdcv = dcv; // wipe out dcv without destroying the members, get its own memory - DCV_init(&dcv, tdcv.size); + DIV_init(&dcv, tdcv.size); } else { // destroy members, but keep memory - DCV_reset(&dcv); + DIV_reset(&dcv); } loop_end: @@ -931,7 +788,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (dsi > 1){ // otherwise dcv equals tcl - DCV_destroy(&dcv); + DIV_destroy(&dcv); } // Return the actual python object - its just a container @@ -939,7 +796,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) if (!dcl){ PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); // Otherwise tdcv would be deallocated by the chunk list - DCV_destroy(&tdcv); + DIV_destroy(&tdcv); error = 1; } else { // Plain copy, don't deallocate From 29fe629dfc11103398f97f0886d79a6f59deeb75 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Oct 2010 18:37:52 +0200 Subject: [PATCH 0106/1392] Intermediate commit, working my way through the code, step by step. Didn't even try to compile it yet --- _delta_apply.c | 183 +++++++++++++++++++++++++++++++++++++------------ stream.py | 13 +--- 2 files changed, 144 insertions(+), 52 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 9768b5264..bd09bc5ed 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -14,6 +14,24 @@ typedef uchar bool; const ull gDIV_grow_by = 100; +// DELTA STREAM ACCESS +/////////////////////// +inline +ull msb_size(const uchar** datap, const uchar* top) +{ + const uchar *data = *datap; + ull cmd, size = 0; + uint i = 0; + do { + cmd = *data++; + size |= (cmd & 0x7f) << i; + i += 7; + } while (cmd & 0x80 && data < top); + *datap = data; + return size; +} + + // DELTA INFO ///////////// typedef struct { @@ -22,6 +40,65 @@ typedef struct { } DeltaInfo; +// TOP LEVEL STREAM INFO +///////////////////////////// +typedef struct { + const uchar* tds; + Py_ssize_t* tdslen; + Py_ssize_t target_size; // size of the target buffer which can hold all data + PyObject* parent_object; +} ToplevelStreamInfo; + + +void TSI_init(ToplevelStreamInfo* info) +{ + info->tds = 0; + info->tdslen = 0; + info->target_size = 0; + info->parent_object = 0; + +} + +void TSI_destroy(ToplevelStreamInfo* info) +{ + if (info->parent_object){ + Py_DECREF(info->parent_object); + info->parent_object = 0; + } else if (info->tds){ + PyMem_Free(info->tds); + } +} + +// initialize our set stream to point to the first chunk +// Fill in the header information, which is the base and target size +void TSI_init_stream(ToplevelStreamInfo* info) +{ + assert(info->tds && info->tdslen) + + // init stream + const uchar* tdsend = info->tds + info->tdslen; + msb_size(&info->tds, tdsend); + info->target_size = msb_size(&info->tds, tdsend); +} + +// duplicate the data currently owned by the parent object drop its refcount +// return 1 on success +bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) +{ + assert(info.parent_object); + + uchar* ptmp = PyMem_Malloc(info.tdslen); + if (!ptmp){ + return 0; + } + memcpy((void*)ptmp, info.tds, info.tdslen); + tds = ptmp; + Py_DECREF(info.parent_object); + info.parent_object = 0; + + return 1; +} + // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -452,7 +529,7 @@ bool DIV_connect_with_base(DeltaInfoVector* tdcv, const DeltaInfoVector* bdcv) typedef struct { PyObject_HEAD // ----------- - DeltaInfoVector vec; + ToplevelStreamInfo istream; } DeltaChunkList; @@ -465,34 +542,20 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) return -1; } - DIV_init(&self->vec, 0); + TSI_init(&self->istream, 0); return 0; } static void DCL_dealloc(DeltaChunkList* self) { - DIV_destroy(&(self->vec)); -} - -static -PyObject* DCL_len(DeltaChunkList* self) -{ - return PyLong_FromUnsignedLongLong(DIV_len(&self->vec)); -} - -static inline -ull DCL_rbound(DeltaChunkList* self) -{ - if (DIV_empty(&self->vec)) - return 0; - return DIV_rbound(&self->vec); + TSI_destroy(&(self->istream)); } static PyObject* DCL_py_rbound(DeltaChunkList* self) { - return PyLong_FromUnsignedLongLong(DCL_rbound(self)); + return PyLong_FromUnsignedLongLong(self->istream->target_size); } // Write using a write function, taking remaining bytes from a base buffer @@ -535,7 +598,6 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) static PyMethodDef DCL_methods[] = { {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, - {"__len__", (PyCFunction)DCL_len, METH_NOARGS, NULL}, {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, {NULL} /* Sentinel */ }; @@ -596,21 +658,6 @@ DeltaChunkList* DCL_new_instance(void) return dcl; } -inline -ull msb_size(const uchar** datap, const uchar* top) -{ - const uchar *data = *datap; - ull cmd, size = 0; - uint i = 0; - do { - cmd = *data++; - size |= (cmd & 0x7f) << i; - i += 7; - } while (cmd & 0x80 && data < top); - *datap = data; - return size; -} - static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) { // obtain iterator @@ -626,22 +673,71 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } DeltaInfoVector dcv; - DeltaInfoVector tdcv; + ToplevelStreamInfo tdsinfo; + TSI_init(&tdsinfo); DIV_init(&dcv, 100); // should be enough to keep the average text file - DIV_init(&tdcv, 0); - unsigned int dsi = 0; - PyObject* ds = 0; + + // GET TOPLEVEL DELTA STREAM int error = 0; - for (ds = PyIter_Next(stream_iter), dsi = 0; ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) + PyObject* ds = 0; + unsigned int dsi = 0; + ds = PyIter_Next(stream_iter); + if (!ds){ + error = 1; + goto _error; + } + + dsi += 1; + tdsinfo.parent_object = PyObject_CallMethod(ds, "read", 0); + if (!PyObject_CheckReadBuffer(tdsinfo.parent_object)){ + Py_DECREF(ds); + error = 1; + goto _error; + } + + PyObject_AsReadBuffer(tdsinfo.parent_object, (const void**)&tdsinfo.tds, &tdsinfo.tdslen); + if (tdslen > pow(2, 32)){ + // parent object is deallocated by info structure + Py_DECREF(ds); + PyErr_SetString(PyExc_RuntimeError("Cannot handle deltas larger than 4GB")); + tdsinfo.tdb = 0; + + error = 1; + goto _error; + } + Py_DECREF(ds); + + // INTEGRATE ANCESTOR DELTA STREAMS + PyObject* db = 0; + TSI_init_stream(&tdsinfo, tdb); + + + for (ds = PyIter_Next(stream_iter); ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) { - PyObject* db = PyObject_CallMethod(ds, "read", 0); + // Its important to initialize this before the next block which can jump + // to code who needs this to exist ! + PyObject* db = 0; + + // When processing the first delta, we know we will have to alter the tds + // Hence we copy it and deallocate the parent object + if (ds == 1) { + if (!TSI_copy_stream_from_object(&tdsinfo)){ + PyErr_SetString(PyExc_RuntimeError, "Could not allocate memory to copy toplevel buffer"); + // info structure takes care of the parent_object + error = 1; + goto loop_end; + } + } + + db = PyObject_CallMethod(ds, "read", 0); if (!PyObject_CheckReadBuffer(db)){ error = 1; PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); goto loop_end; } + // Fill the stream info structure const uchar* data; Py_ssize_t dlen; PyObject_AsReadBuffer(db, (const void**)&data, &dlen); @@ -778,10 +874,13 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } }// END for each stream object - if (dsi == 0 && ! error){ + if (dsi == 0){ PyErr_SetString(PyExc_ValueError, "No streams provided"); } + +_error: + if (stream_iter != dstreams){ Py_DECREF(stream_iter); } @@ -800,7 +899,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) error = 1; } else { // Plain copy, don't deallocate - dcl->vec = tdcv; + dcl->istream = tdsinfo; } if (error){ diff --git a/stream.py b/stream.py index 38c86dae7..0d8972898 100644 --- a/stream.py +++ b/stream.py @@ -349,7 +349,7 @@ def _set_cache_too_slow_without_c(self, attr): # call len directly, as the (optional) c version doesn't implement the sequence # protocol - if dcl.__len__() == 0: + if dcl.rbound() == 0: self._size = 0 self._mm_target = allocate_memory(0) return @@ -367,15 +367,6 @@ def _set_cache_too_slow_without_c(self, attr): self._mm_target.seek(0) - def _set_cache_(self, attr): - """Determine which version to use depending on the configuration of the deltas - :note: we are only called if we have the performance module""" - # otherwise it depends on the amount of memory to shift around - if len(self._dstreams) > 1 and self._bstream.size < 150000: - return self._set_cache_too_slow_without_c(attr) - else: - return self._set_cache_brute_(attr) - def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" @@ -456,6 +447,8 @@ def _set_cache_brute_(self, attr): #{ Configuration if not has_perf_mod: _set_cache_ = _set_cache_brute_ + else: + _set_cache_ = _set_cache_too_slow_without_c #} END configuration From 2414fd48969bf5bff510c8ded83edd7b4900626b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Oct 2010 21:06:17 +0200 Subject: [PATCH 0107/1392] Brutally made code compile, most of the major functions are still commented out, but it should just be a matter of time until its back and working --- _delta_apply.c | 324 ++++++++++++++++++++++--------------------------- 1 file changed, 142 insertions(+), 182 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index bd09bc5ed..abeaed5f1 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -32,31 +32,24 @@ ull msb_size(const uchar** datap, const uchar* top) } -// DELTA INFO -///////////// -typedef struct { - uint dso; // delta stream offset - uint to; // target offset (cache) -} DeltaInfo; - - // TOP LEVEL STREAM INFO ///////////////////////////// typedef struct { - const uchar* tds; - Py_ssize_t* tdslen; - Py_ssize_t target_size; // size of the target buffer which can hold all data - PyObject* parent_object; + const uchar *tds; + Py_ssize_t tdslen; // size of tds in bytes + Py_ssize_t target_size; // size of the target buffer which can hold all data + uint numChunks; // amount of chunks in the delta stream + PyObject *parent_object; } ToplevelStreamInfo; void TSI_init(ToplevelStreamInfo* info) { - info->tds = 0; + info->tds = NULL; info->tdslen = 0; + info->numChunks = 0; info->target_size = 0; info->parent_object = 0; - } void TSI_destroy(ToplevelStreamInfo* info) @@ -65,7 +58,7 @@ void TSI_destroy(ToplevelStreamInfo* info) Py_DECREF(info->parent_object); info->parent_object = 0; } else if (info->tds){ - PyMem_Free(info->tds); + PyMem_Free((void*)info->tds); } } @@ -73,7 +66,7 @@ void TSI_destroy(ToplevelStreamInfo* info) // Fill in the header information, which is the base and target size void TSI_init_stream(ToplevelStreamInfo* info) { - assert(info->tds && info->tdslen) + assert(info->tds && info->tdslen); // init stream const uchar* tdsend = info->tds + info->tdslen; @@ -85,16 +78,16 @@ void TSI_init_stream(ToplevelStreamInfo* info) // return 1 on success bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) { - assert(info.parent_object); + assert(info->parent_object); - uchar* ptmp = PyMem_Malloc(info.tdslen); + uchar* ptmp = PyMem_Malloc(info->tdslen); if (!ptmp){ return 0; } - memcpy((void*)ptmp, info.tds, info.tdslen); - tds = ptmp; - Py_DECREF(info.parent_object); - info.parent_object = 0; + memcpy((void*)ptmp, info->tds, info->tdslen); + info->tds = ptmp; + Py_DECREF(info->parent_object); + info->parent_object = 0; return 1; } @@ -152,11 +145,21 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec // DELTA CHUNK VECTOR ///////////////////// + +// DELTA INFO +///////////// +typedef struct { + uint dso; // delta stream offset + uint to; // target offset (cache) +} DeltaInfo; + + typedef struct { - DeltaInfo* mem; // Memory - const uchar* dstream; // pointer to delta stream we index - Py_ssize_t size; // Size in DeltaInfos - Py_ssize_t reserved_size; // Reserve in DeltaInfos + DeltaInfo *mem; // Memory + uint di_last_size; // size of the last element - we can't compute it using the next bound + const uchar *dstream; // pointer to delta stream we index - its borrowed + Py_ssize_t size; // Amount of DeltaInfos + Py_ssize_t reserved_size; // Reserved amount of DeltaInfos } DeltaInfoVector; @@ -164,7 +167,7 @@ typedef struct { // Reserve enough memory to hold the given amount of delta chunks // Return 1 on success // NOTE: added a minimum allocation to assure reallocation is not done -// just for a single additional entry. DCVs change often, and reallocs are expensive +// just for a single additional entry. DIVs change often, and reallocs are expensive inline int DIV_reserve_memory(DeltaInfoVector* vec, uint num_dc) { @@ -219,18 +222,19 @@ int DIV_init(DeltaInfoVector* vec, ull initial_size) vec->mem = NULL; vec->size = 0; vec->reserved_size = 0; + vec->di_last_size = 0; return DIV_grow_by(vec, initial_size); } inline -ull DIV_len(const DeltaInfoVector* vec) +Py_ssize_t DIV_len(const DeltaInfoVector* vec) { return vec->size; } inline -ull DIV_lbound(const DeltaInfoVector* vec) +uint DIV_lbound(const DeltaInfoVector* vec) { assert(vec->size && vec->mem); return vec->mem->to; @@ -251,18 +255,6 @@ DeltaInfo* DIV_last(const DeltaInfoVector* vec) return DIV_get(vec, vec->size-1); } -inline -ull DIV_rbound(const DeltaInfoVector* vec) -{ - return DC_rbound(DIV_last(vec)); -} - -inline -ull DIV_size(const DeltaInfoVector* vec) -{ - return DIV_rbound(vec) - DIV_lbound(vec); -} - inline int DIV_empty(const DeltaInfoVector* vec) { @@ -285,19 +277,24 @@ DeltaInfo* DIV_first(const DeltaInfoVector* vec) return vec->mem; } +// return rbound offset in bytes. We use information contained in the +// vec to do that +inline +uint DIV_info_rbound(const DeltaInfoVector* vec, const DeltaInfo* di) +{ + if (DIV_last(vec) == di){ + return di->to + vec->di_last_size; + } else { + return (di+1)->to; + } +} + void DIV_destroy(DeltaInfoVector* vec) { if (vec->mem){ #ifdef DEBUG fprintf(stderr, "Freeing %p\n", (void*)vec->mem); #endif - - const DeltaInfo* end = &vec->mem[vec->size]; - DeltaInfo* i; - for(i = vec->mem; i < end; i++){ - DC_destroy(i); - } - PyMem_Free(vec->mem); vec->size = 0; vec->reserved_size = 0; @@ -313,21 +310,13 @@ void DIV_forget_members(DeltaInfoVector* vec) vec->size = 0; } -// Reset the vector so that its size will be zero, and its members will -// have been deallocated properly. +// Reset the vector so that its size will be zero // It will keep its memory though, and hence can be filled again inline void DIV_reset(DeltaInfoVector* vec) { if (vec->size == 0) return; - - DeltaInfo* dc = DIV_first(vec); - const DeltaInfo* dcend = DIV_end(vec); - for(;dc < dcend; dc++){ - DC_destroy(dc); - } - vec->size = 0; } @@ -363,7 +352,7 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) dc = vec->mem + mid; if (dc->to > ofs){ hi = mid; - } else if ((DC_rbound(dc) > ofs) | (dc->to == ofs)) { + } else if ((DIV_info_rbound(vec, dc) > ofs) | (dc->to == ofs)) { return dc; } else { lo = mid + 1; @@ -378,7 +367,7 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) inline uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) { - uint num_dc = 0; + /*uint num_dc = 0; DeltaInfo* cdc = DIV_closest_chunk(src, ofs); // partial overlap @@ -404,7 +393,9 @@ uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) } } - return num_dc; + return num_dc;*/ + assert(0); // TODO + return 0; } // Write a slice as defined by its absolute offset in bytes and its size into the given @@ -414,8 +405,9 @@ uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) inline uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull size) { + /* assert(DIV_lbound(src) <= ofs); - assert((ofs + size) <= DIV_rbound(src)); + assert((ofs + size) <= DIV_last(src)->to + src->di_last_size); DeltaInfo* cdc = DIV_closest_chunk(src, ofs); uint num_chunks = 0; @@ -450,13 +442,17 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull assert(size == 0); return num_chunks; + */ + assert(0); // TODO + return 0; } -// Take slices of bdcv into the corresponding area of the tdcv, which is the topmost -// delta to apply. -bool DIV_connect_with_base(DeltaInfoVector* tdcv, const DeltaInfoVector* bdcv) +// Take slices of div into the corresponding area of the tsi, which is the topmost +// delta to apply. +bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { + /* uint *const offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); if (!offset_array){ return 0; @@ -521,6 +517,9 @@ bool DIV_connect_with_base(DeltaInfoVector* tdcv, const DeltaInfoVector* bdcv) PyMem_Free(offset_array); return 1; + */ + assert(0); // TODO + return 0; } // DELTA CHUNK LIST (PYTHON) @@ -542,7 +541,7 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) return -1; } - TSI_init(&self->istream, 0); + TSI_init(&self->istream); return 0; } @@ -555,13 +554,14 @@ void DCL_dealloc(DeltaChunkList* self) static PyObject* DCL_py_rbound(DeltaChunkList* self) { - return PyLong_FromUnsignedLongLong(self->istream->target_size); + return PyLong_FromUnsignedLongLong(self->istream.target_size); } // Write using a write function, taking remaining bytes from a base buffer static PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { + /* PyObject* pybuf = 0; PyObject* writeproc = 0; if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ @@ -593,6 +593,9 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) } Py_DECREF(tmpargs); + */ + // TODO + assert(0); Py_RETURN_NONE; } @@ -653,11 +656,51 @@ DeltaChunkList* DCL_new_instance(void) assert(dcl); DCL_init(dcl, 0, 0); - assert(dcl->vec.size == 0); - assert(dcl->vec.mem == NULL); return dcl; } +// Read the next delta chunk from the given stream and advance it +// dc will contain the parsed information, its offset must be set by +// the previous call of next_delta_info, which implies it should remain the +// same instance between the calls. +// Return 1 on success, 0 on failure +inline +bool next_delta_info(const uchar** dstream, DeltaChunk* dc) +{ + const uchar* data = *dstream; + const char cmd = *data++; + + if (cmd & 0x80) + { + unsigned long cp_off = 0, cp_size = 0; + if (cmd & 0x01) cp_off = *data++; + if (cmd & 0x02) cp_off |= (*data++ << 8); + if (cmd & 0x04) cp_off |= (*data++ << 16); + if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); + if (cmd & 0x10) cp_size = *data++; + if (cmd & 0x20) cp_size |= (*data++ << 8); + if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cp_size == 0) cp_size = 0x10000; + + dc->to += dc->ts; + dc->data = 0; + dc->so = cp_off; + dc->ts = cp_size; + + } else if (cmd) { + // Just share the data + dc->to += dc->ts; + dc->data = data; + dc->ts = cmd; + dc->so = 0; + } else { + PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + return 0; + } + + return 1; +} + static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) { // obtain iterator @@ -672,16 +715,16 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) stream_iter = dstreams; } - DeltaInfoVector dcv; + DeltaInfoVector div; ToplevelStreamInfo tdsinfo; TSI_init(&tdsinfo); - DIV_init(&dcv, 100); // should be enough to keep the average text file + DIV_init(&div, 100); // should be enough to keep the average text file // GET TOPLEVEL DELTA STREAM int error = 0; PyObject* ds = 0; - unsigned int dsi = 0; + unsigned int dsi = 0; // delta stream index we process ds = PyIter_Next(stream_iter); if (!ds){ error = 1; @@ -697,11 +740,11 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } PyObject_AsReadBuffer(tdsinfo.parent_object, (const void**)&tdsinfo.tds, &tdsinfo.tdslen); - if (tdslen > pow(2, 32)){ + if (tdsinfo.tdslen > pow(2, 32)){ // parent object is deallocated by info structure Py_DECREF(ds); - PyErr_SetString(PyExc_RuntimeError("Cannot handle deltas larger than 4GB")); - tdsinfo.tdb = 0; + PyErr_SetString(PyExc_RuntimeError, "Cannot handle deltas larger than 4GB"); + tdsinfo.parent_object = 0; error = 1; goto _error; @@ -709,11 +752,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(ds); // INTEGRATE ANCESTOR DELTA STREAMS - PyObject* db = 0; - TSI_init_stream(&tdsinfo, tdb); + TSI_init_stream(&tdsinfo); - for (ds = PyIter_Next(stream_iter); ds != NULL; ++dsi, ds = PyIter_Next(stream_iter)) + for (ds = PyIter_Next(stream_iter); ds != NULL; ds = PyIter_Next(stream_iter), ++dsi) { // Its important to initialize this before the next block which can jump // to code who needs this to exist ! @@ -721,7 +763,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // When processing the first delta, we know we will have to alter the tds // Hence we copy it and deallocate the parent object - if (ds == 1) { + if (dsi == 1) { if (!TSI_copy_stream_from_object(&tdsinfo)){ PyErr_SetString(PyExc_RuntimeError, "Could not allocate memory to copy toplevel buffer"); // info structure takes care of the parent_object @@ -744,125 +786,45 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const uchar* dend = data + dlen; // read header - const ull base_size = msb_size(&data, dend); + msb_size(&data, dend); const ull target_size = msb_size(&data, dend); // Assume good compression for the adds const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); - DIV_reserve_memory(&dcv, approx_num_cmds); + DIV_reserve_memory(&div, approx_num_cmds); // parse command stream - ull tbw = 0; // Amount of target bytes written - bool is_shared_data = dsi != 0; - bool is_first_run = dsi == 0; + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); assert(data < dend); while (data < dend) { - const char cmd = *data++; - - if (cmd & 0x80) - { - unsigned long cp_off = 0, cp_size = 0; - if (cmd & 0x01) cp_off = *data++; - if (cmd & 0x02) cp_off |= (*data++ << 8); - if (cmd & 0x04) cp_off |= (*data++ << 16); - if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); - if (cmd & 0x10) cp_size = *data++; - if (cmd & 0x20) cp_size |= (*data++ << 8); - if (cmd & 0x40) cp_size |= (*data++ << 16); - if (cp_size == 0) cp_size = 0x10000; - - const unsigned long rbound = cp_off + cp_size; - if (rbound < cp_size || - rbound > base_size){ - // this really shouldn't happen - error = 1; - assert(0); - break; - } - - DC_init(DIV_append(&dcv), tbw, cp_size, cp_off); - tbw += cp_size; - - } else if (cmd) { - // Compression reduces fragmentation though, which is why we do it - // in all cases. - // It makes the more sense the more consecutive add-chunks we have, - // its more likely in big deltas, for big binary files - const uchar* add_start = data - 1; - const uchar* add_end = dend; - ull num_bytes = cmd; - data += cmd; - ull num_chunks = 1; - while (data < dend){ - //while (0){ - const char c = *data; - if (c & 0x80){ - add_end = data; - break; - } else { - data += 1 + c; // advance by 1 to skip add cmd - num_bytes += c; - num_chunks += 1; - } - } - - #ifdef DEBUG - assert(add_end - add_start > 0); - if (num_chunks > 1){ - fprintf(stderr, "Compression: got %i bytes of %i chunks\n", (int)num_bytes, (int)num_chunks); - } - #endif - - DeltaChunk* dc = DIV_append(&dcv); - DC_init(dc, tbw, num_bytes, 0); - - // gather the data, or (possibly) share single blocks - if (num_chunks > 1){ - uchar* dcdata = PyMem_Malloc(num_bytes); - while (add_start < add_end){ - const char bytes = *add_start++; - memcpy((void*)dcdata, (void*)add_start, bytes); - dcdata += bytes; - add_start += bytes; - } - DC_set_data_with_ownership(dc, dcdata-num_bytes); - } else { - DC_set_data(dc, data - cmd, cmd, is_shared_data); - } - - tbw += num_bytes; - } else { + if (next_delta_info(&data, &dc)){ + // TODO + assert(0); + } else { error = 1; - PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); goto loop_end; } }// END handle command opcodes - if (tbw != target_size){ + + if (DC_rbound(&dc) != target_size){ PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); error = 1; } - if (!is_first_run){ - if (!DIV_connect_with_base(&tdcv, &dcv)){ - error = 1; - } + if (!DIV_connect_with_base(&tdsinfo, &div)){ + error = 1; } - + #ifdef DEBUG - fprintf(stderr, "tdcv->size = %i, tdcv->reserved_size = %i\n", (int)tdcv.size, (int)tdcv.reserved_size); - fprintf(stderr, "dcv->size = %i, dcv->reserved_size = %i\n", (int)dcv.size, (int)dcv.reserved_size); + fprintf(stderr, "tdsinfo->len = %i\n", (int)tdsinfo.tdslen); + fprintf(stderr, "div->size = %i, div->reserved_size = %i\n", (int)div.size, (int)div.reserved_size); #endif - if (is_first_run){ - tdcv = dcv; - // wipe out dcv without destroying the members, get its own memory - DIV_init(&dcv, tdcv.size); - } else { - // destroy members, but keep memory - DIV_reset(&dcv); - } + // destroy members, but keep memory + DIV_reset(&div); loop_end: // perform cleanup @@ -885,20 +847,18 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) Py_DECREF(stream_iter); } - if (dsi > 1){ - // otherwise dcv equals tcl - DIV_destroy(&dcv); - } + + DIV_destroy(&div); // Return the actual python object - its just a container DeltaChunkList* dcl = DCL_new_instance(); if (!dcl){ PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); - // Otherwise tdcv would be deallocated by the chunk list - DIV_destroy(&tdcv); + // Otherwise tdsinfo would be deallocated by the chunk list + TSI_destroy(&tdsinfo); error = 1; } else { - // Plain copy, don't deallocate + // Plain copy, transfer ownership to dcl dcl->istream = tdsinfo; } From 9e62c5481fefcc9e8adf0b6387952e214295223c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 00:52:43 +0200 Subject: [PATCH 0108/1392] Worked my way up to re-encoding delta chunks, connect_with method still needs some work, intermediate commit --- _delta_apply.c | 309 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 243 insertions(+), 66 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index abeaed5f1..d35cb7ebc 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -8,6 +8,7 @@ typedef unsigned long long ull; typedef unsigned int uint; typedef unsigned char uchar; +typedef unsigned short ushort; typedef uchar bool; // Constants @@ -35,10 +36,11 @@ ull msb_size(const uchar** datap, const uchar* top) // TOP LEVEL STREAM INFO ///////////////////////////// typedef struct { - const uchar *tds; + const uchar *tds; // Toplevel delta stream + const uchar *cstart; // start of the chunks Py_ssize_t tdslen; // size of tds in bytes Py_ssize_t target_size; // size of the target buffer which can hold all data - uint numChunks; // amount of chunks in the delta stream + uint num_chunks; // amount of chunks in the delta stream PyObject *parent_object; } ToplevelStreamInfo; @@ -46,8 +48,9 @@ typedef struct { void TSI_init(ToplevelStreamInfo* info) { info->tds = NULL; + info->cstart = NULL; info->tdslen = 0; - info->numChunks = 0; + info->num_chunks = 0; info->target_size = 0; info->parent_object = 0; } @@ -62,18 +65,37 @@ void TSI_destroy(ToplevelStreamInfo* info) } } +inline +const uchar* TSI_end(ToplevelStreamInfo* info) +{ + return info->tds + info->tdslen; +} + +inline +const uchar* TSI_first(ToplevelStreamInfo* info) +{ + return info->cstart; +} + +// set the stream, and initialize it // initialize our set stream to point to the first chunk // Fill in the header information, which is the base and target size -void TSI_init_stream(ToplevelStreamInfo* info) +inline +void TSI_set_stream(ToplevelStreamInfo* info, const uchar* stream) { + info->tds = stream; + info->cstart = stream; + assert(info->tds && info->tdslen); // init stream - const uchar* tdsend = info->tds + info->tdslen; - msb_size(&info->tds, tdsend); - info->target_size = msb_size(&info->tds, tdsend); + const uchar* tdsend = TSI_end(info); + msb_size(&info->cstart, tdsend); // base size + info->target_size = msb_size(&info->cstart, tdsend); } + + // duplicate the data currently owned by the parent object drop its refcount // return 1 on success bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) @@ -86,12 +108,29 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) } memcpy((void*)ptmp, info->tds, info->tdslen); info->tds = ptmp; + info->cstart = ptmp; Py_DECREF(info->parent_object); info->parent_object = 0; return 1; } +// make sure we have the given amount of memory available. This will change +// our official length in bytes right away, its up to the caller +// to do something useful with the freed space +// Return true on success +bool TSI_resize(ToplevelStreamInfo* info, uint num_bytes) +{ + assert(info->tds); + if (num_bytes <= info->tdslen){ + return 1; + } + info->tds = PyMem_Realloc((void*)info->tds, num_bytes); + info->cstart = info->tds; + + return info->tds != NULL; +} + // DELTA CHUNK //////////////// // Internal Delta Chunk Objects @@ -99,8 +138,8 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) // The data pointer is always shared typedef struct { ull to; - ull ts; - ull so; + uint ts; + uint so; const uchar* data; } DeltaChunk; @@ -141,9 +180,66 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec } +// Encode the information in the given delta chunk and write the byte-stream +// into the given output stream +inline +void DC_encode_to(const DeltaChunk* dc, uchar** pout) +{ + uchar* out = *pout; + if (dc->data){ + *out++ = (uchar)dc->ts; + memcpy(out, dc->data, dc->ts); + out += dc->ts; + } else { + uchar i = 0x80; + uchar* op = out++; + uint moff = dc->so; + uint msize = dc->ts; + + if (moff & 0x000000ff) + *out++ = moff >> 0, i |= 0x01; + if (moff & 0x0000ff00) + *out++ = moff >> 8, i |= 0x02; + if (moff & 0x00ff0000) + *out++ = moff >> 16, i |= 0x04; + if (moff & 0xff000000) + *out++ = moff >> 24, i |= 0x08; + + if (msize & 0x00ff) + *out++ = msize >> 0, i |= 0x10; + if (msize & 0xff00) + *out++ = msize >> 8, i |= 0x20; + + *op = i; + } + *pout = out; +} + +// Return: amount of bytes one would need to encode dc +inline +ushort DC_count_encode_bytes(const DeltaChunk* dc) +{ + if (dc->data){ + return 1 + dc->ts; // cmd byte + actual data bytes + } else { + ushort c = 1; // cmd byte + uint ts = dc->ts; + ull to = dc->to; + + // offset + c += to & 0x000000FF; + c += to & 0x0000FF00; + c += to & 0x00FF0000; + c += to & 0xFF000000; + + // size - max size is 0x10000, its encoded with 0 size bits + c += ts & 0x000000FF; + c += ts & 0x0000FF00; + + return c; + } +} -// DELTA CHUNK VECTOR -///////////////////// // DELTA INFO @@ -154,10 +250,13 @@ typedef struct { } DeltaInfo; +// DELTA INFO VECTOR +////////////////////// + typedef struct { - DeltaInfo *mem; // Memory - uint di_last_size; // size of the last element - we can't compute it using the next bound - const uchar *dstream; // pointer to delta stream we index - its borrowed + DeltaInfo *mem; // Memory for delta infos + uint di_last_size; // size of the last element - we can't compute it using the next bound + const uchar *dstream; // borrowed ointer to delta stream we index Py_ssize_t size; // Amount of DeltaInfos Py_ssize_t reserved_size; // Reserved amount of DeltaInfos } DeltaInfoVector; @@ -220,6 +319,7 @@ int DIV_grow_by(DeltaInfoVector* vec, uint num_dc) int DIV_init(DeltaInfoVector* vec, ull initial_size) { vec->mem = NULL; + vec->dstream = NULL; vec->size = 0; vec->reserved_size = 0; vec->di_last_size = 0; @@ -289,6 +389,24 @@ uint DIV_info_rbound(const DeltaInfoVector* vec, const DeltaInfo* di) } } +// return size of the given delta info item +inline +uint DIV_info_size2(const DeltaInfoVector* vec, const DeltaInfo* di, const DeltaInfo const* veclast) +{ + if (veclast == di){ + return vec->di_last_size; + } else { + return (di+1)->to - di->to; + } +} + +// return size of the given delta info item +inline +uint DIV_info_size(const DeltaInfoVector* vec, const DeltaInfo* di) +{ + return DIV_info_size2(vec, di, DIV_last(vec)); +} + void DIV_destroy(DeltaInfoVector* vec) { if (vec->mem){ @@ -344,16 +462,16 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) ull lo = 0; ull hi = vec->size; ull mid; - DeltaInfo* dc; + DeltaInfo* di; while (lo < hi) { mid = (lo + hi) / 2; - dc = vec->mem + mid; - if (dc->to > ofs){ + di = vec->mem + mid; + if (di->to > ofs){ hi = mid; - } else if ((DIV_info_rbound(vec, dc) > ofs) | (dc->to == ofs)) { - return dc; + } else if ((DIV_info_rbound(vec, di) > ofs) | (di->to == ofs)) { + return di; } else { lo = mid + 1; } @@ -362,40 +480,58 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) return DIV_last(vec); } +// forward declaration +const uchar* next_delta_info(const uchar*, DeltaChunk*); -// Return the amount of chunks a slice at the given spot would have +// Return the amount of chunks a slice at the given spot would have, as well as +// its size in bytes it would have if the possibly partial chunks would be encoded +// The bytes will be added inline -uint DIV_count_slice_chunks(const DeltaInfoVector* src, ull ofs, ull size) +uint DIV_count_slice_chunks_and_bytes(const DeltaInfoVector* src, ull ofs, ull size, uint* out_bytes) { - /*uint num_dc = 0; - DeltaInfo* cdc = DIV_closest_chunk(src, ofs); + uint num_dc = 0; + DeltaInfo* cdi = DIV_closest_chunk(src, ofs); + + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); // partial overlap - if (cdc->to != ofs) { - const ull relofs = ofs - cdc->to; - size -= cdc->ts - relofs < size ? cdc->ts - relofs : size; + if (cdi->to != ofs) { + const ull relofs = ofs - cdi->to; + const uint cdisize = DIV_info_size(src, cdi); + size -= cdisize - relofs < size ? cdisize - relofs : size; num_dc += 1; - cdc += 1; + cdi += 1; + + // get the size in bytes the info would have + next_delta_info(src->dstream + cdi->dso, &dc); + *out_bytes += DC_count_encode_bytes(&dc); if (size == 0){ return num_dc; } } - const DeltaInfo* vecend = DIV_end(src); - for( ;(cdc < vecend) && size; ++cdc){ + const DeltaInfo const* vecend = DIV_end(src); + const DeltaInfo const* veclast = DIV_last(src); + for( ;(cdi < vecend) && size; ++cdi){ num_dc += 1; - if (cdc->ts < size) { - size -= cdc->ts; + + const uint cdisize = DIV_info_size2(src, cdi, veclast); + + next_delta_info(src->dstream + cdi->dso, &dc); + *out_bytes += DC_count_encode_bytes(&dc); + + if (cdisize < size) { + size -= cdisize; } else { size = 0; break; } } - return num_dc;*/ - assert(0); // TODO - return 0; + *out_bytes += 0; + return num_dc; } // Write a slice as defined by its absolute offset in bytes and its size into the given @@ -447,40 +583,47 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull return 0; } - // Take slices of div into the corresponding area of the tsi, which is the topmost // delta to apply. bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { - /* - uint *const offset_array = PyMem_Malloc(tdcv->size * sizeof(uint)); + assert(tsi->num_chunks); + + uint *const offset_array = PyMem_Malloc(tsi->num_chunks * sizeof(uint)); if (!offset_array){ return 0; } uint* pofs = offset_array; uint num_addchunks = 0; + uint num_addbytes = 0; - DeltaInfo* dc = DIV_first(tdcv); - const DeltaInfo* dcend = DIV_end(tdcv); + const uchar* data = TSI_first(tsi); + const uchar const* dend = TSI_end(tsi); + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); // OFFSET RUN - for (;dc < dcend; dc++, pofs++) + for (;data < dend; pofs++) { // Data chunks don't need processing *pofs = num_addchunks; - if (dc->data){ + data = next_delta_info(data, &dc); + + if (dc.data){ continue; } // offset the next chunk by the amount of chunks in the slice // - 1, because we replace our own chunk - num_addchunks += DIV_count_slice_chunks(bdcv, dc->so, dc->ts) - 1; + num_addchunks += DIV_count_slice_chunks_and_bytes(div, dc.so, dc.ts, &num_addbytes) - 1; + assert(num_addbytes); } + /* // reserve enough memory to hold all the new chunks // reinit pointers, array could have been reallocated - DIV_reserve_memory(tdcv, tdcv->size + num_addchunks); + TSI_resize(tsis, tsi->tdslen + num_addbytes); dc = DIV_last(tdcv); dcend = DIV_first(tdcv) - 1; @@ -514,12 +657,10 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) tdc->to += relofs; } } - + */ PyMem_Free(offset_array); return 1; - */ - assert(0); // TODO - return 0; + } // DELTA CHUNK LIST (PYTHON) @@ -663,23 +804,22 @@ DeltaChunkList* DCL_new_instance(void) // dc will contain the parsed information, its offset must be set by // the previous call of next_delta_info, which implies it should remain the // same instance between the calls. -// Return 1 on success, 0 on failure +// Return the altered uchar pointer, reassign it to the input data inline -bool next_delta_info(const uchar** dstream, DeltaChunk* dc) +const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) { - const uchar* data = *dstream; const char cmd = *data++; if (cmd & 0x80) { - unsigned long cp_off = 0, cp_size = 0; + uint cp_off = 0, cp_size = 0; if (cmd & 0x01) cp_off = *data++; if (cmd & 0x02) cp_off |= (*data++ << 8); if (cmd & 0x04) cp_off |= (*data++ << 16); if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); if (cmd & 0x10) cp_size = *data++; if (cmd & 0x20) cp_size |= (*data++ << 8); - if (cmd & 0x40) cp_size |= (*data++ << 16); + if (cmd & 0x40) cp_size |= (*data++ << 16); // this should never get hit with current deltas ... if (cp_size == 0) cp_size = 0x10000; dc->to += dc->ts; @@ -695,10 +835,34 @@ bool next_delta_info(const uchar** dstream, DeltaChunk* dc) dc->so = 0; } else { PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); - return 0; + return NULL; } - return 1; + return data; +} + +// Return amount of chunks encoded in the given delta stream +// If read_header is True, then the header msb chunks will be read first. +// Otherwise, the stream is assumed to be scrubbed one past the header +uint compute_chunk_count(const uchar* data, const uchar* dend, bool read_header) +{ + // read header + if (read_header){ + msb_size(&data, dend); + msb_size(&data, dend); + } + + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); + uint num_chunks = 0; + + while (data < dend) + { + data = next_delta_info(data, &dc); + num_chunks += 1; + }// END handle command opcodes + + return num_chunks; } static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) @@ -751,10 +915,10 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } Py_DECREF(ds); - // INTEGRATE ANCESTOR DELTA STREAMS - TSI_init_stream(&tdsinfo); - + // let it officially know, and initialize its internal state + TSI_set_stream(&tdsinfo, tdsinfo.tds); + // INTEGRATE ANCESTOR DELTA STREAMS for (ds = PyIter_Next(stream_iter); ds != NULL; ds = PyIter_Next(stream_iter), ++dsi) { // Its important to initialize this before the next block which can jump @@ -770,6 +934,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) error = 1; goto loop_end; } + + tdsinfo.num_chunks = compute_chunk_count(tdsinfo.cstart, TSI_end(&tdsinfo), 0); } db = PyObject_CallMethod(ds, "read", 0); @@ -783,37 +949,48 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const uchar* data; Py_ssize_t dlen; PyObject_AsReadBuffer(db, (const void**)&data, &dlen); - const uchar* dend = data + dlen; + const uchar const* dstart = data; + const uchar const* dend = data + dlen; + div.dstream = dstart; + + if (dlen > pow(2, 32)){ + error = 1; + PyErr_SetString(PyExc_RuntimeError, "Cannot currently handle deltas larger than 4GB"); + goto loop_end; + } - // read header + // READ HEADER msb_size(&data, dend); const ull target_size = msb_size(&data, dend); - // Assume good compression for the adds - const uint approx_num_cmds = ((dlen / 3) / 10) + (((dlen / 3) * 2) / (2+2+1)); - DIV_reserve_memory(&div, approx_num_cmds); + DIV_reserve_memory(&div, compute_chunk_count(data, dend, 0)); // parse command stream DeltaChunk dc; + DeltaInfo* di = 0; // temporary pointer DC_init(&dc, 0, 0, 0, NULL); assert(data < dend); while (data < dend) { - if (next_delta_info(&data, &dc)){ - // TODO - assert(0); + di = DIV_append(&div); + di->dso = data - dstart; + if ((data = next_delta_info(data, &dc))){ + di->to = dc.to; } else { error = 1; goto loop_end; } }// END handle command opcodes + // finalize information + div.di_last_size = dc.ts; + if (DC_rbound(&dc) != target_size){ PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); error = 1; } - + if (!DIV_connect_with_base(&tdsinfo, &div)){ error = 1; } From 8693a7e37af03cd5d36f36fcc5ed01e938798905 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 12:26:07 +0200 Subject: [PATCH 0109/1392] Implemented connect_with which includes all the slicing functions, which now operate on the delta stream data directly, its yet to be tested though, and I am afraid of this --- _delta_apply.c | 207 ++++++++++++++++++++++++++++--------------------- 1 file changed, 118 insertions(+), 89 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index d35cb7ebc..5f1d9c379 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -106,9 +106,12 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) if (!ptmp){ return 0; } + uint ofs = (uint)(info->cstart - info->tds); memcpy((void*)ptmp, info->tds, info->tdslen); + info->tds = ptmp; - info->cstart = ptmp; + info->cstart = ptmp + ofs; + Py_DECREF(info->parent_object); info->parent_object = 0; @@ -125,8 +128,10 @@ bool TSI_resize(ToplevelStreamInfo* info, uint num_bytes) if (num_bytes <= info->tdslen){ return 1; } + uint ofs = (uint)(info->cstart - info->tds); info->tds = PyMem_Realloc((void*)info->tds, num_bytes); - info->cstart = info->tds; + info->tdslen = num_bytes; + info->cstart = info->tds + ofs; return info->tds != NULL; } @@ -182,19 +187,21 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec // Encode the information in the given delta chunk and write the byte-stream // into the given output stream +// It will be copied into the given bounds, the given size must be the final size +// and work with the given relative offset - hence the bounds are assumed to be +// correct and to fit within the unaltered dc inline -void DC_encode_to(const DeltaChunk* dc, uchar** pout) +void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) { uchar* out = *pout; if (dc->data){ - *out++ = (uchar)dc->ts; - memcpy(out, dc->data, dc->ts); - out += dc->ts; + *out++ = (uchar)size; + memcpy(out, dc->data+ofs, size); + out += size; } else { uchar i = 0x80; uchar* op = out++; - uint moff = dc->so; - uint msize = dc->ts; + uint moff = dc->so+ofs; if (moff & 0x000000ff) *out++ = moff >> 0, i |= 0x01; @@ -205,10 +212,10 @@ void DC_encode_to(const DeltaChunk* dc, uchar** pout) if (moff & 0xff000000) *out++ = moff >> 24, i |= 0x08; - if (msize & 0x00ff) - *out++ = msize >> 0, i |= 0x10; - if (msize & 0xff00) - *out++ = msize >> 8, i |= 0x20; + if (size & 0x00ff) + *out++ = size >> 0, i |= 0x10; + if (size & 0xff00) + *out++ = size >> 8, i |= 0x20; *op = i; } @@ -224,13 +231,13 @@ ushort DC_count_encode_bytes(const DeltaChunk* dc) } else { ushort c = 1; // cmd byte uint ts = dc->ts; - ull to = dc->to; + ull so = dc->so; // offset - c += to & 0x000000FF; - c += to & 0x0000FF00; - c += to & 0x00FF0000; - c += to & 0xFF000000; + c += so & 0x000000FF; + c += so & 0x0000FF00; + c += so & 0x00FF0000; + c += so & 0xFF000000; // size - max size is 0x10000, its encoded with 0 size bits c += ts & 0x000000FF; @@ -485,13 +492,15 @@ const uchar* next_delta_info(const uchar*, DeltaChunk*); // Return the amount of chunks a slice at the given spot would have, as well as // its size in bytes it would have if the possibly partial chunks would be encoded -// The bytes will be added +// and added to the spot marked by sdc inline -uint DIV_count_slice_chunks_and_bytes(const DeltaInfoVector* src, ull ofs, ull size, uint* out_bytes) +uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) { - uint num_dc = 0; + uint num_bytes = 0; DeltaInfo* cdi = DIV_closest_chunk(src, ofs); + + DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -499,63 +508,72 @@ uint DIV_count_slice_chunks_and_bytes(const DeltaInfoVector* src, ull ofs, ull s if (cdi->to != ofs) { const ull relofs = ofs - cdi->to; const uint cdisize = DIV_info_size(src, cdi); - size -= cdisize - relofs < size ? cdisize - relofs : size; - num_dc += 1; - cdi += 1; + const uint actual_size = cdisize - relofs < size ? cdisize - relofs : size; + size -= actual_size; // get the size in bytes the info would have next_delta_info(src->dstream + cdi->dso, &dc); - *out_bytes += DC_count_encode_bytes(&dc); + dc.so += relofs; + dc.ts = actual_size; + num_bytes += DC_count_encode_bytes(&dc); + + cdi += 1; if (size == 0){ - return num_dc; + return num_bytes; } } const DeltaInfo const* vecend = DIV_end(src); - const DeltaInfo const* veclast = DIV_last(src); - for( ;(cdi < vecend) && size; ++cdi){ - num_dc += 1; - - const uint cdisize = DIV_info_size2(src, cdi, veclast); - + for( ;cdi < vecend; ++cdi){ next_delta_info(src->dstream + cdi->dso, &dc); - *out_bytes += DC_count_encode_bytes(&dc); - if (cdisize < size) { - size -= cdisize; + if (dc.ts < size) { + num_bytes += DC_count_encode_bytes(&dc); + size -= dc.ts; } else { + dc.ts = size; + num_bytes += DC_count_encode_bytes(&dc); size = 0; break; } } - *out_bytes += 0; - return num_dc; + assert(size == 0); + return num_bytes; } // Write a slice as defined by its absolute offset in bytes and its size into the given -// destination memory. The individual chunks written will be a deep copy of the source -// data chunks +// destination memory. The individual chunks written will be a byte copy of the source +// data chunk stream // Return: number of chunks in the slice inline -uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull size) +uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint size) { - /* - assert(DIV_lbound(src) <= ofs); - assert((ofs + size) <= DIV_last(src)->to + src->di_last_size); + assert(DIV_lbound(src) <= tofs); + assert((tofs + size) <= DIV_last(src)->to + src->di_last_size); - DeltaInfo* cdc = DIV_closest_chunk(src, ofs); + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); + + DeltaInfo* cdi = DIV_closest_chunk(src, tofs); uint num_chunks = 0; // partial overlap - if (cdc->to != ofs) { - const ull relofs = ofs - cdc->to; - DC_offset_copy_to(cdc, dest, relofs, cdc->ts - relofs < size ? cdc->ts - relofs : size); - cdc += 1; - size -= dest->ts; - dest += 1; // must be here, we are reading the size ! + if (cdi->to != tofs) { + const uint relofs = tofs - cdi->to; + next_delta_info(src->dstream + cdi->dso, &dc); + const uint cdisize = dc.ts; + const uint actual_size = cdisize - relofs < size ? cdisize - relofs : size; + + size -= actual_size; + + // adjust dc proportions + + DC_encode_to(&dc, &dest, relofs, actual_size); + num_chunks += 1; + cdi += 1; if (size == 0){ return num_chunks; @@ -563,14 +581,18 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull } const DeltaInfo* vecend = DIV_end(src); - for( ;(cdc < vecend) && size; ++cdc) + for( ;cdi < vecend; ++cdi) { num_chunks += 1; - if (cdc->ts < size) { - DC_copy_to(cdc, dest++); - size -= cdc->ts; + next_delta_info(src->dstream + cdi->dso, &dc); + if (dc.ts < size) { + // Full copy would be possible, but the final length of the dstream + // needs to be used as well to know how many bytes to copy + // TODO: make a DIV_ function for this + DC_encode_to(&dc, &dest, 0, dc.ts); + size -= dc.ts; } else { - DC_offset_copy_to(cdc, dest, 0, size); + DC_encode_to(&dc, &dest, 0, size); size = 0; break; } @@ -578,36 +600,44 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, DeltaInfo* dest, ull ofs, ull assert(size == 0); return num_chunks; - */ - assert(0); // TODO - return 0; } + // Take slices of div into the corresponding area of the tsi, which is the topmost // delta to apply. bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { assert(tsi->num_chunks); - uint *const offset_array = PyMem_Malloc(tsi->num_chunks * sizeof(uint)); + typedef struct { + uint bofs; // byte-offset of delta stream + uint dofs; // delta stream offset relative to tsi->cstart + } OffsetInfo; + + + OffsetInfo *const offset_array = PyMem_Malloc(tsi->num_chunks * sizeof(OffsetInfo)); if (!offset_array){ return 0; } - uint* pofs = offset_array; - uint num_addchunks = 0; + OffsetInfo* pofs = offset_array; uint num_addbytes = 0; const uchar* data = TSI_first(tsi); + const uchar* prev_data = data; const uchar const* dend = TSI_end(tsi); + DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); // OFFSET RUN - for (;data < dend; pofs++) + for (;data < dend; pofs++, prev_data = data) { + + pofs->bofs = num_addbytes; + pofs->dofs = (uint)(prev_data - data); + // Data chunks don't need processing - *pofs = num_addchunks; data = next_delta_info(data, &dc); if (dc.data){ @@ -615,49 +645,48 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) } // offset the next chunk by the amount of chunks in the slice - // - 1, because we replace our own chunk - num_addchunks += DIV_count_slice_chunks_and_bytes(div, dc.so, dc.ts, &num_addbytes) - 1; - assert(num_addbytes); + // - N, because we replace our own chunk's bytes + num_addbytes += DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); } - /* - // reserve enough memory to hold all the new chunks - // reinit pointers, array could have been reallocated - TSI_resize(tsis, tsi->tdslen + num_addbytes); - dc = DIV_last(tdcv); - dcend = DIV_first(tdcv) - 1; - // now, that we have our pointers with the old size - tdcv->size += num_addchunks; + + // reserve enough memory to hold all the new chunks + TSI_resize(tsi, tsi->tdslen + num_addbytes); + const OffsetInfo const* pofs_start = offset_array - 1; + const OffsetInfo* cpofs; + uchar* ds; // pointer into the delta stream + const uchar* nds; // next pointer, used for size retrieving the size + uint num_addchunks = 0; // total amount of chunks added // Insert slices, from the end to the beginning, which allows memcpy // to be used, with a little help of the offset array - for (pofs -= 1; dc > dcend; dc--, pofs-- ) + for (cpofs = pofs - 1; cpofs > pofs_start; cpofs--) { + ds = (uchar*)(tsi->cstart + cpofs->dofs); + nds = next_delta_info(ds, &dc); + // Data chunks don't need processing - const uint ofs = *pofs; - if (dc->data){ + if (dc.data){ // NOTE: could peek the preceeding chunks to figure out whether they are // all just moved by ofs. In that case, they can move as a whole! // tests showed that this is very rare though, even in huge deltas, so its // not worth the extra effort - if (ofs){ - memcpy((void*)(dc + ofs), (void*)dc, sizeof(DeltaInfo)); + if (pofs->bofs){ + memcpy((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); } continue; } - // Copy Chunks, and move their target offset into place - // As we could override dc when slicing, we get the data here - const ull relofs = dc->to - dc->so; - - DeltaInfo* tdc = dc + ofs; - DeltaInfo* tdcend = tdc + DIV_copy_slice_to(bdcv, tdc, dc->so, dc->ts); - for(;tdc < tdcend; tdc++){ - tdc->to += relofs; - } + // Copy Chunks - target offset is determined by their location and size + // hence it doesn't need specific adjustment + // -1 chunks because we overwrite our own chunk ( by not copying it ) + num_addchunks += DIV_copy_slice_to(div, ds + cpofs->bofs, dc.so, dc.ts); + num_addchunks -= 1; } - */ + + tsi->num_chunks += num_addchunks; + PyMem_Free(offset_array); return 1; @@ -823,7 +852,7 @@ const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) if (cp_size == 0) cp_size = 0x10000; dc->to += dc->ts; - dc->data = 0; + dc->data = NULL; dc->so = cp_off; dc->ts = cp_size; From c16de40fccf84d066194d2bf95dea56aa76dafd1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 14:34:00 +0200 Subject: [PATCH 0110/1392] Implemented apply - there are still some issues to work out though --- _delta_apply.c | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 5f1d9c379..6d7c4c9d0 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -165,7 +165,6 @@ ull DC_rbound(const DeltaChunk* dc) } // Apply -// TODO: remove, just left it for reference inline void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) { @@ -252,7 +251,7 @@ ushort DC_count_encode_bytes(const DeltaChunk* dc) // DELTA INFO ///////////// typedef struct { - uint dso; // delta stream offset + uint dso; // delta stream offset, relative to the very start of the stream uint to; // target offset (cache) } DeltaInfo; @@ -281,10 +280,6 @@ int DIV_reserve_memory(DeltaInfoVector* vec, uint num_dc) return 1; } - if (num_dc - vec->reserved_size < 10){ - num_dc += gDIV_grow_by; - } - #ifdef DEBUG bool was_null = vec->mem == NULL; #endif @@ -529,6 +524,8 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) next_delta_info(src->dstream + cdi->dso, &dc); if (dc.ts < size) { + // TODO: could just count size of the delta chunk in the stream instead + // of reencoding num_bytes += DC_count_encode_bytes(&dc); size -= dc.ts; } else { @@ -731,7 +728,7 @@ PyObject* DCL_py_rbound(DeltaChunkList* self) static PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { - /* + PyObject* pybuf = 0; PyObject* writeproc = 0; if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ @@ -749,23 +746,24 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) return NULL; } - const DeltaChunk* i = self->vec.mem; - const DeltaChunk* end = DIV_end(&self->vec); - - const uchar* data; - Py_ssize_t dlen; - PyObject_AsReadBuffer(pybuf, (const void**)&data, &dlen); + const uchar* base; + Py_ssize_t baselen; + PyObject_AsReadBuffer(pybuf, (const void**)&base, &baselen); PyObject* tmpargs = PyTuple_New(1); - for(; i < end; i++){ - DC_apply(i, data, writeproc, tmpargs); + const uchar* data = TSI_first(&self->istream); + const uchar const* dend = TSI_end(&self->istream); + + DeltaChunk dc; + DC_init(&dc, 0, 0, 0, NULL); + + while (data < dend){ + data = next_delta_info(data, &dc); + DC_apply(&dc, base, writeproc, tmpargs); } Py_DECREF(tmpargs); - */ - // TODO - assert(0); Py_RETURN_NONE; } @@ -911,7 +909,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DeltaInfoVector div; ToplevelStreamInfo tdsinfo; TSI_init(&tdsinfo); - DIV_init(&div, 100); // should be enough to keep the average text file + DIV_init(&div, 0); // GET TOPLEVEL DELTA STREAM @@ -1020,13 +1018,17 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) error = 1; } + #ifdef DEBUG + fprintf(stderr, "Before Connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + #endif + if (!DIV_connect_with_base(&tdsinfo, &div)){ error = 1; } #ifdef DEBUG - fprintf(stderr, "tdsinfo->len = %i\n", (int)tdsinfo.tdslen); - fprintf(stderr, "div->size = %i, div->reserved_size = %i\n", (int)div.size, (int)div.reserved_size); + fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); #endif // destroy members, but keep memory From dc85535de371762503496313f5e753d61997dcfd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 16:22:59 +0200 Subject: [PATCH 0111/1392] Its much closer to a working state now, but still not quite there. Probably just a small thing --- _delta_apply.c | 72 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 6d7c4c9d0..65e949a04 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -57,6 +57,7 @@ void TSI_init(ToplevelStreamInfo* info) void TSI_destroy(ToplevelStreamInfo* info) { + fprintf(stderr, "TSI_destroy: %p\n", info); if (info->parent_object){ Py_DECREF(info->parent_object); info->parent_object = 0; @@ -164,6 +165,12 @@ ull DC_rbound(const DeltaChunk* dc) return dc->to + dc->ts; } +inline +void DC_print(const DeltaChunk* dc, const char* prefix) +{ + fprintf(stderr, "%s-dc: to = %i, ts = %i, so = %i, data = %p\n", prefix, (int)dc->to, dc->ts, dc->so, dc->data); +} + // Apply inline void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) @@ -179,6 +186,8 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec assert(0); } + DC_print(dc, "DC_apply"); + // tuple steals reference, and will take care about the deallocation PyObject_Call(writer, tmpargs, NULL); @@ -233,14 +242,14 @@ ushort DC_count_encode_bytes(const DeltaChunk* dc) ull so = dc->so; // offset - c += so & 0x000000FF; - c += so & 0x0000FF00; - c += so & 0x00FF0000; - c += so & 0xFF000000; + c += (so & 0x000000FF) > 0; + c += (so & 0x0000FF00) > 0; + c += (so & 0x00FF0000) > 0; + c += (so & 0xFF000000) > 0; // size - max size is 0x10000, its encoded with 0 size bits - c += ts & 0x000000FF; - c += ts & 0x0000FF00; + c += (ts & 0x000000FF) > 0; + c += (ts & 0x0000FF00) > 0; return c; } @@ -413,7 +422,7 @@ void DIV_destroy(DeltaInfoVector* vec) { if (vec->mem){ #ifdef DEBUG - fprintf(stderr, "Freeing %p\n", (void*)vec->mem); + fprintf(stderr, "DIV_destroy: %p\n", (void*)vec->mem); #endif PyMem_Free(vec->mem); vec->size = 0; @@ -494,8 +503,6 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) uint num_bytes = 0; DeltaInfo* cdi = DIV_closest_chunk(src, ofs); - - DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -503,13 +510,13 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) if (cdi->to != ofs) { const ull relofs = ofs - cdi->to; const uint cdisize = DIV_info_size(src, cdi); - const uint actual_size = cdisize - relofs < size ? cdisize - relofs : size; - size -= actual_size; + const uint max_size = cdisize - relofs < size ? cdisize - relofs : size; + size -= max_size; // get the size in bytes the info would have next_delta_info(src->dstream + cdi->dso, &dc); dc.so += relofs; - dc.ts = actual_size; + dc.ts = max_size; num_bytes += DC_count_encode_bytes(&dc); cdi += 1; @@ -547,8 +554,9 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) inline uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint size) { + fprintf(stderr, "copy slice: ofs = %i, size = %i\n", (int)tofs, size); assert(DIV_lbound(src) <= tofs); - assert((tofs + size) <= DIV_last(src)->to + src->di_last_size); + assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -556,18 +564,22 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s DeltaInfo* cdi = DIV_closest_chunk(src, tofs); uint num_chunks = 0; +#ifdef DEBUG + const uchar* deststart = dest; +#endif + // partial overlap if (cdi->to != tofs) { const uint relofs = tofs - cdi->to; next_delta_info(src->dstream + cdi->dso, &dc); const uint cdisize = dc.ts; - const uint actual_size = cdisize - relofs < size ? cdisize - relofs : size; + const uint max_size = cdisize - relofs < size ? cdisize - relofs : size; - size -= actual_size; + size -= max_size; // adjust dc proportions - DC_encode_to(&dc, &dest, relofs, actual_size); + DC_encode_to(&dc, &dest, relofs, max_size); num_chunks += 1; cdi += 1; @@ -580,6 +592,7 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s const DeltaInfo* vecend = DIV_end(src); for( ;cdi < vecend; ++cdi) { + fprintf(stderr, "copy slice: cdi: to = %i, dso = %i\n", (int)cdi->to, (int)cdi->dso); num_chunks += 1; next_delta_info(src->dstream + cdi->dso, &dc); if (dc.ts < size) { @@ -595,6 +608,10 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s } } +#ifdef DEBUG + fprintf(stderr, "copy slice: Wrote %i bytes\n", (int)(dest - deststart)); +#endif + assert(size == 0); return num_chunks; } @@ -619,6 +636,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) OffsetInfo* pofs = offset_array; uint num_addbytes = 0; + uint dofs = 0; const uchar* data = TSI_first(tsi); const uchar* prev_data = data; @@ -627,16 +645,19 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); + // OFFSET RUN for (;data < dend; pofs++, prev_data = data) { - pofs->bofs = num_addbytes; - pofs->dofs = (uint)(prev_data - data); - - // Data chunks don't need processing data = next_delta_info(data, &dc); + pofs->dofs = dofs; + dofs += (uint)(data-prev_data); + + fprintf(stderr, "pofs->bofs = %i, ->dofs = %i\n", pofs->bofs, pofs->dofs); + DC_print(&dc, "count-run"); + // Data chunks don't need processing if (dc.data){ continue; } @@ -646,6 +667,8 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) num_addbytes += DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); } + fprintf(stderr, "num_addbytes = %i\n", num_addbytes); + assert(DC_rbound(&dc) == tsi->target_size); // reserve enough memory to hold all the new chunks @@ -655,6 +678,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) uchar* ds; // pointer into the delta stream const uchar* nds; // next pointer, used for size retrieving the size uint num_addchunks = 0; // total amount of chunks added + DC_init(&dc, 0, 0, 0, NULL); // Insert slices, from the end to the beginning, which allows memcpy // to be used, with a little help of the offset array @@ -682,6 +706,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) num_addchunks -= 1; } + fprintf(stderr, "num_addchunks = %i\n", num_addchunks); tsi->num_chunks += num_addchunks; PyMem_Free(offset_array); @@ -860,6 +885,8 @@ const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) dc->data = data; dc->ts = cmd; dc->so = 0; + + data += cmd; } else { PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); return NULL; @@ -993,8 +1020,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) DIV_reserve_memory(&div, compute_chunk_count(data, dend, 0)); // parse command stream - DeltaChunk dc; DeltaInfo* di = 0; // temporary pointer + DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); assert(data < dend); @@ -1019,7 +1046,9 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } #ifdef DEBUG + fprintf(stderr, "------------ Stream %i --------\n ", (int)dsi); fprintf(stderr, "Before Connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); #endif if (!DIV_connect_with_base(&tdsinfo, &div)){ @@ -1028,7 +1057,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) #ifdef DEBUG fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); - fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); #endif // destroy members, but keep memory From c077f3f9ac56aa1019273c7ea548ae047e5974f6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 16:28:28 +0200 Subject: [PATCH 0112/1392] now it appears to work completely, still plenty of debug printing though --- _delta_apply.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_delta_apply.c b/_delta_apply.c index 65e949a04..350a2493d 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -693,7 +693,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) // all just moved by ofs. In that case, they can move as a whole! // tests showed that this is very rare though, even in huge deltas, so its // not worth the extra effort - if (pofs->bofs){ + if (cpofs->bofs){ memcpy((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); } continue; From 28263c97bca171d84b4f4ea022d6638076b903a4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 17:32:31 +0200 Subject: [PATCH 0113/1392] When using it with deeper chains, it can still crash as it can actually (try to) shrink a chunk. This is currently not handled. In that case, we had to virtually move everything x bytes, which should be much like an offset --- _delta_apply.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 350a2493d..2632fbb27 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -57,7 +57,10 @@ void TSI_init(ToplevelStreamInfo* info) void TSI_destroy(ToplevelStreamInfo* info) { +#ifdef DEBUG fprintf(stderr, "TSI_destroy: %p\n", info); +#endif + if (info->parent_object){ Py_DECREF(info->parent_object); info->parent_object = 0; @@ -129,6 +132,10 @@ bool TSI_resize(ToplevelStreamInfo* info, uint num_bytes) if (num_bytes <= info->tdslen){ return 1; } + +#ifdef DEBUG + fprintf(stderr, "TSI_resize: to %i bytes\n", num_bytes); +#endif uint ofs = (uint)(info->cstart - info->tds); info->tds = PyMem_Realloc((void*)info->tds, num_bytes); info->tdslen = num_bytes; @@ -186,7 +193,6 @@ void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObjec assert(0); } - DC_print(dc, "DC_apply"); // tuple steals reference, and will take care about the deallocation PyObject_Call(writer, tmpargs, NULL); @@ -554,7 +560,6 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) inline uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint size) { - fprintf(stderr, "copy slice: ofs = %i, size = %i\n", (int)tofs, size); assert(DIV_lbound(src) <= tofs); assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); @@ -564,10 +569,6 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s DeltaInfo* cdi = DIV_closest_chunk(src, tofs); uint num_chunks = 0; -#ifdef DEBUG - const uchar* deststart = dest; -#endif - // partial overlap if (cdi->to != tofs) { const uint relofs = tofs - cdi->to; @@ -592,7 +593,6 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s const DeltaInfo* vecend = DIV_end(src); for( ;cdi < vecend; ++cdi) { - fprintf(stderr, "copy slice: cdi: to = %i, dso = %i\n", (int)cdi->to, (int)cdi->dso); num_chunks += 1; next_delta_info(src->dstream + cdi->dso, &dc); if (dc.ts < size) { @@ -608,10 +608,6 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s } } -#ifdef DEBUG - fprintf(stderr, "copy slice: Wrote %i bytes\n", (int)(dest - deststart)); -#endif - assert(size == 0); return num_chunks; } @@ -624,7 +620,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) assert(tsi->num_chunks); typedef struct { - uint bofs; // byte-offset of delta stream + int bofs; // byte-offset of delta stream uint dofs; // delta stream offset relative to tsi->cstart } OffsetInfo; @@ -635,7 +631,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) } OffsetInfo* pofs = offset_array; - uint num_addbytes = 0; + int num_addbytes = 0; uint dofs = 0; const uchar* data = TSI_first(tsi); @@ -651,12 +647,10 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { pofs->bofs = num_addbytes; data = next_delta_info(data, &dc); + assert(data); pofs->dofs = dofs; dofs += (uint)(data-prev_data); - fprintf(stderr, "pofs->bofs = %i, ->dofs = %i\n", pofs->bofs, pofs->dofs); - DC_print(&dc, "count-run"); - // Data chunks don't need processing if (dc.data){ continue; @@ -667,7 +661,12 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) num_addbytes += DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); } - fprintf(stderr, "num_addbytes = %i\n", num_addbytes); + /* + uint i = 0; + for (; i < tsi->num_chunks; i++){ + fprintf(stderr, "%i: bofs: %i, dofs: %i\n", i, offset_array[i].bofs, offset_array[i].dofs); + } + */ assert(DC_rbound(&dc) == tsi->target_size); @@ -695,6 +694,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) // not worth the extra effort if (cpofs->bofs){ memcpy((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); + // memmove((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); } continue; } @@ -706,7 +706,6 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) num_addchunks -= 1; } - fprintf(stderr, "num_addchunks = %i\n", num_addchunks); tsi->num_chunks += num_addchunks; PyMem_Free(offset_array); From 1a57dc133ec31c409819b2b40866529ed95a555b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 17:53:12 +0200 Subject: [PATCH 0114/1392] Well, the virtual movement doesn't work - the algorithm really worked only with a fixed chunk size. Now the only chance we have is to allocate an appropriately sized buffer, and work through it directly. This makes things easier, and will make things work \! --- _delta_apply.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 2632fbb27..9df0191e4 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -620,7 +620,7 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) assert(tsi->num_chunks); typedef struct { - int bofs; // byte-offset of delta stream + uint bofs; // byte-offset of delta stream uint dofs; // delta stream offset relative to tsi->cstart } OffsetInfo; @@ -631,7 +631,8 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) } OffsetInfo* pofs = offset_array; - int num_addbytes = 0; + uint num_addbytes = 0; + int bytes = 0; uint dofs = 0; const uchar* data = TSI_first(tsi); @@ -658,15 +659,16 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) // offset the next chunk by the amount of chunks in the slice // - N, because we replace our own chunk's bytes - num_addbytes += DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); + bytes = DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); + // if we shrink in size, compensate this by moving the start virtually + // + if (bytes < 0){ + fprintf(stderr, "hit negative bytes: %i\n", bytes); + tsi->cstart += abs(bytes); + } + num_addbytes += abs(bytes); } - /* - uint i = 0; - for (; i < tsi->num_chunks; i++){ - fprintf(stderr, "%i: bofs: %i, dofs: %i\n", i, offset_array[i].bofs, offset_array[i].dofs); - } - */ assert(DC_rbound(&dc) == tsi->target_size); @@ -701,8 +703,8 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) // Copy Chunks - target offset is determined by their location and size // hence it doesn't need specific adjustment - // -1 chunks because we overwrite our own chunk ( by not copying it ) num_addchunks += DIV_copy_slice_to(div, ds + cpofs->bofs, dc.so, dc.ts); + // -1 chunks because we overwrite our own chunk ( by not copying it ) num_addchunks -= 1; } From a7892bc7ebc99d2fe726287cc69716a3494b5ea2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 20:25:01 +0200 Subject: [PATCH 0115/1392] goooosh, it took so long to find a tiny nasty bug ... aarggghhh, lots of debug printing still in there, ... this one better be faster than anything else \! --- _delta_apply.c | 182 ++++++++++++++++++++++++++----------------------- 1 file changed, 95 insertions(+), 87 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 9df0191e4..71d5b5f8c 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -15,6 +15,7 @@ typedef uchar bool; const ull gDIV_grow_by = 100; + // DELTA STREAM ACCESS /////////////////////// inline @@ -63,10 +64,14 @@ void TSI_destroy(ToplevelStreamInfo* info) if (info->parent_object){ Py_DECREF(info->parent_object); - info->parent_object = 0; + info->parent_object = NULL; } else if (info->tds){ PyMem_Free((void*)info->tds); } + info->tds = NULL; + info->cstart = NULL; + info->tdslen = 0; + info->num_chunks = 0; } inline @@ -122,26 +127,21 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) return 1; } -// make sure we have the given amount of memory available. This will change -// our official length in bytes right away, its up to the caller -// to do something useful with the freed space -// Return true on success -bool TSI_resize(ToplevelStreamInfo* info, uint num_bytes) +// Transfer ownership of the given stream into our instance. The amount of chunks +// remains the same, and needs to be set by the caller +void TSI_replace_stream(ToplevelStreamInfo* info, const uchar* stream, uint streamlen) { - assert(info->tds); - if (num_bytes <= info->tdslen){ - return 1; - } + assert(info->parent_object == 0); + fprintf(stderr, "TSI_replace_stream\n"); -#ifdef DEBUG - fprintf(stderr, "TSI_resize: to %i bytes\n", num_bytes); -#endif uint ofs = (uint)(info->cstart - info->tds); - info->tds = PyMem_Realloc((void*)info->tds, num_bytes); - info->tdslen = num_bytes; + if (info->tds){ + PyMem_Free((void*)info->tds); + } + info->tds = stream; info->cstart = info->tds + ofs; + info->tdslen = streamlen; - return info->tds != NULL; } // DELTA CHUNK @@ -156,6 +156,9 @@ typedef struct { const uchar* data; } DeltaChunk; +// forward declarations +const uchar* next_delta_info(const uchar*, DeltaChunk*); + inline void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, const uchar* data) { @@ -208,6 +211,8 @@ inline void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) { uchar* out = *pout; + DC_print(dc, "DC_encode_to"); + fprintf(stderr, "DC_encode_to: ofs = %i, size = %i\n" , ofs, size); if (dc->data){ *out++ = (uchar)size; memcpy(out, dc->data+ofs, size); @@ -233,6 +238,18 @@ void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) *op = i; } + +#ifdef DEBUG + DeltaChunk mdc; + DC_init(&mdc, 0, 0, 0, NULL); + next_delta_info(*pout, &mdc); + assert(mdc.ts == size); + if (mdc.data) + assert(mdc.data); + else + assert(mdc.so == dc->so+ofs); +#endif + *pout = out; } @@ -497,8 +514,6 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) return DIV_last(vec); } -// forward declaration -const uchar* next_delta_info(const uchar*, DeltaChunk*); // Return the amount of chunks a slice at the given spot would have, as well as // its size in bytes it would have if the possibly partial chunks would be encoded @@ -558,10 +573,11 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) // data chunk stream // Return: number of chunks in the slice inline -uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint size) +uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint size) { assert(DIV_lbound(src) <= tofs); assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); + fprintf(stderr, "copy_slice: ofs = %i, size = %i\n", (int)tofs, size); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -573,14 +589,12 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s if (cdi->to != tofs) { const uint relofs = tofs - cdi->to; next_delta_info(src->dstream + cdi->dso, &dc); - const uint cdisize = dc.ts; - const uint max_size = cdisize - relofs < size ? cdisize - relofs : size; + const uint max_size = dc.ts - relofs < size ? dc.ts - relofs : size; size -= max_size; // adjust dc proportions - - DC_encode_to(&dc, &dest, relofs, max_size); + DC_encode_to(&dc, dest, relofs, max_size); num_chunks += 1; cdi += 1; @@ -599,10 +613,10 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar* dest, ull tofs, uint s // Full copy would be possible, but the final length of the dstream // needs to be used as well to know how many bytes to copy // TODO: make a DIV_ function for this - DC_encode_to(&dc, &dest, 0, dc.ts); + DC_encode_to(&dc, dest, 0, dc.ts); size -= dc.ts; } else { - DC_encode_to(&dc, &dest, 0, size); + DC_encode_to(&dc, dest, 0, size); size = 0; break; } @@ -619,98 +633,90 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { assert(tsi->num_chunks); - typedef struct { - uint bofs; // byte-offset of delta stream - uint dofs; // delta stream offset relative to tsi->cstart - } OffsetInfo; - - - OffsetInfo *const offset_array = PyMem_Malloc(tsi->num_chunks * sizeof(OffsetInfo)); - if (!offset_array){ - return 0; - } - - OffsetInfo* pofs = offset_array; - uint num_addbytes = 0; - int bytes = 0; - uint dofs = 0; + uint num_bytes = 0; const uchar* data = TSI_first(tsi); - const uchar* prev_data = data; - const uchar const* dend = TSI_end(tsi); + const uchar* dend = TSI_end(tsi); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); - // OFFSET RUN - for (;data < dend; pofs++, prev_data = data) + // COMPUTE SIZE OF TARGET STREAM + ///////////////////////////////// + for (;data < dend;) { - pofs->bofs = num_addbytes; data = next_delta_info(data, &dc); - assert(data); - pofs->dofs = dofs; - dofs += (uint)(data-prev_data); + DC_print(&dc, "count"); // Data chunks don't need processing if (dc.data){ + num_bytes += 1 + dc.ts; continue; } - // offset the next chunk by the amount of chunks in the slice - // - N, because we replace our own chunk's bytes - bytes = DIV_count_slice_bytes(div, dc.so, dc.ts) - (data - prev_data); - // if we shrink in size, compensate this by moving the start virtually - // - if (bytes < 0){ - fprintf(stderr, "hit negative bytes: %i\n", bytes); - tsi->cstart += abs(bytes); - } - num_addbytes += abs(bytes); + num_bytes += DIV_count_slice_bytes(div, dc.so, dc.ts); } - assert(DC_rbound(&dc) == tsi->target_size); - // reserve enough memory to hold all the new chunks - TSI_resize(tsi, tsi->tdslen + num_addbytes); - const OffsetInfo const* pofs_start = offset_array - 1; - const OffsetInfo* cpofs; - uchar* ds; // pointer into the delta stream - const uchar* nds; // next pointer, used for size retrieving the size - uint num_addchunks = 0; // total amount of chunks added + // GET NEW DELTA BUFFER + //////////////////////// + uchar *const dstream = PyMem_Malloc(num_bytes); + if (!dstream){ + return 0; + } + + + data = TSI_first(tsi); + const uchar *ndata = data; + dend = TSI_end(tsi); + + uint num_chunks = 0; + uchar* ds = dstream; DC_init(&dc, 0, 0, 0, NULL); - // Insert slices, from the end to the beginning, which allows memcpy - // to be used, with a little help of the offset array - for (cpofs = pofs - 1; cpofs > pofs_start; cpofs--) + // pick slices from the delta and put them into the new stream + for (; data < dend; data = ndata) { - ds = (uchar*)(tsi->cstart + cpofs->dofs); - nds = next_delta_info(ds, &dc); + ndata = next_delta_info(data, &dc); + + DC_print(&dc, "slice"); // Data chunks don't need processing if (dc.data){ - // NOTE: could peek the preceeding chunks to figure out whether they are - // all just moved by ofs. In that case, they can move as a whole! - // tests showed that this is very rare though, even in huge deltas, so its - // not worth the extra effort - if (cpofs->bofs){ - memcpy((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); - // memmove((void*)(ds + cpofs->bofs), (void*)ds, nds - ds); - } + // just copy it over + memcpy((void*)ds, (void*)data, ndata - data); + ds += ndata - data; + num_chunks += 1; continue; } - // Copy Chunks - target offset is determined by their location and size - // hence it doesn't need specific adjustment - num_addchunks += DIV_copy_slice_to(div, ds + cpofs->bofs, dc.so, dc.ts); - // -1 chunks because we overwrite our own chunk ( by not copying it ) - num_addchunks -= 1; + // Copy Chunks + num_chunks += DIV_copy_slice_to(div, &ds, dc.so, dc.ts); } + assert(ds - dstream == num_bytes); + assert(num_chunks >= tsi->num_chunks); + assert(DC_rbound(&dc) == tsi->target_size); + + // finally, replace the streams + TSI_replace_stream(tsi, dstream, num_bytes); + tsi->cstart = dstream; // we have NO header ! + assert(tsi->tds == dstream); + tsi->num_chunks = num_chunks; - tsi->num_chunks += num_addchunks; +#ifdef DEBUG + data = TSI_first(tsi); + dend = TSI_end(tsi); + + DC_init(&dc, 0, 0, 0, NULL); + + while (data < dend){ + data = next_delta_info(data, &dc); + DC_print(&dc, "debug"); + } +#endif - PyMem_Free(offset_array); return 1; } @@ -754,6 +760,7 @@ PyObject* DCL_py_rbound(DeltaChunkList* self) static PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { + fprintf(stderr, "DCL_apply\n"); PyObject* pybuf = 0; PyObject* writeproc = 0; @@ -890,6 +897,7 @@ const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) data += cmd; } else { PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); + assert(0); return NULL; } @@ -1048,7 +1056,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) #ifdef DEBUG fprintf(stderr, "------------ Stream %i --------\n ", (int)dsi); - fprintf(stderr, "Before Connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + fprintf(stderr, "Before Connect: tdsinfo: num_chunks = %i, bytelen = %i, target_size = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen, (int)tdsinfo.target_size); fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); #endif From 81a96250f1758844a1c95f0293083c18c4ce98dc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 20:37:39 +0200 Subject: [PATCH 0116/1392] Removed all debug code, it now runs about as fast as the previous version, but with less memory, still slower than the brute force version though --- _delta_apply.c | 37 +++---------------------------------- 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 71d5b5f8c..068d89a17 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -132,7 +132,6 @@ bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) void TSI_replace_stream(ToplevelStreamInfo* info, const uchar* stream, uint streamlen) { assert(info->parent_object == 0); - fprintf(stderr, "TSI_replace_stream\n"); uint ofs = (uint)(info->cstart - info->tds); if (info->tds){ @@ -211,8 +210,6 @@ inline void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) { uchar* out = *pout; - DC_print(dc, "DC_encode_to"); - fprintf(stderr, "DC_encode_to: ofs = %i, size = %i\n" , ofs, size); if (dc->data){ *out++ = (uchar)size; memcpy(out, dc->data+ofs, size); @@ -239,17 +236,6 @@ void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) *op = i; } -#ifdef DEBUG - DeltaChunk mdc; - DC_init(&mdc, 0, 0, 0, NULL); - next_delta_info(*pout, &mdc); - assert(mdc.ts == size); - if (mdc.data) - assert(mdc.data); - else - assert(mdc.so == dc->so+ofs); -#endif - *pout = out; } @@ -577,7 +563,6 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint { assert(DIV_lbound(src) <= tofs); assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); - fprintf(stderr, "copy_slice: ofs = %i, size = %i\n", (int)tofs, size); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -647,7 +632,6 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) for (;data < dend;) { data = next_delta_info(data, &dc); - DC_print(&dc, "count"); // Data chunks don't need processing if (dc.data){ @@ -681,8 +665,6 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) { ndata = next_delta_info(data, &dc); - DC_print(&dc, "slice"); - // Data chunks don't need processing if (dc.data){ // just copy it over @@ -705,17 +687,6 @@ bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) assert(tsi->tds == dstream); tsi->num_chunks = num_chunks; -#ifdef DEBUG - data = TSI_first(tsi); - dend = TSI_end(tsi); - - DC_init(&dc, 0, 0, 0, NULL); - - while (data < dend){ - data = next_delta_info(data, &dc); - DC_print(&dc, "debug"); - } -#endif return 1; @@ -760,8 +731,6 @@ PyObject* DCL_py_rbound(DeltaChunkList* self) static PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { - fprintf(stderr, "DCL_apply\n"); - PyObject* pybuf = 0; PyObject* writeproc = 0; if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ @@ -1056,8 +1025,8 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) #ifdef DEBUG fprintf(stderr, "------------ Stream %i --------\n ", (int)dsi); - fprintf(stderr, "Before Connect: tdsinfo: num_chunks = %i, bytelen = %i, target_size = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen, (int)tdsinfo.target_size); - fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i\n", (int)div.size, (int)div.reserved_size, (int)dlen); + fprintf(stderr, "Before Connect: tdsinfo: num_chunks = %i, bytelen = %i KiB, target_size = %i KiB\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen/1000, (int)tdsinfo.target_size/1000); + fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i KiB\n", (int)div.size, (int)div.reserved_size, (int)dlen/1000); #endif if (!DIV_connect_with_base(&tdsinfo, &div)){ @@ -1065,7 +1034,7 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) } #ifdef DEBUG - fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen); + fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i KiB\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen/1000); #endif // destroy members, but keep memory From ca829e0b341dd5c3ae1408b24702f2c75db6ec73 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 21:12:42 +0200 Subject: [PATCH 0117/1392] now byte-copying a few chunks where possible when slicing, which gives a few percent of performance --- _delta_apply.c | 20 +++++++++----------- stream.py | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/_delta_apply.c b/_delta_apply.c index 068d89a17..e99a803be 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -534,13 +534,12 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) } const DeltaInfo const* vecend = DIV_end(src); + const uchar* nstream; for( ;cdi < vecend; ++cdi){ - next_delta_info(src->dstream + cdi->dso, &dc); + nstream = next_delta_info(src->dstream + cdi->dso, &dc); if (dc.ts < size) { - // TODO: could just count size of the delta chunk in the stream instead - // of reencoding - num_bytes += DC_count_encode_bytes(&dc); + num_bytes += nstream - (src->dstream + cdi->dso); size -= dc.ts; } else { dc.ts = size; @@ -589,16 +588,15 @@ uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint } } - const DeltaInfo* vecend = DIV_end(src); - for( ;cdi < vecend; ++cdi) + const uchar* dstream = src->dstream + cdi->dso; + const uchar* nstream = dstream; + for( ; nstream; dstream = nstream) { num_chunks += 1; - next_delta_info(src->dstream + cdi->dso, &dc); + nstream = next_delta_info(dstream, &dc); if (dc.ts < size) { - // Full copy would be possible, but the final length of the dstream - // needs to be used as well to know how many bytes to copy - // TODO: make a DIV_ function for this - DC_encode_to(&dc, dest, 0, dc.ts); + memcpy(*dest, dstream, nstream - dstream); + *dest += nstream - dstream; size -= dc.ts; } else { DC_encode_to(&dc, dest, 0, size); diff --git a/stream.py b/stream.py index 0d8972898..f522bd318 100644 --- a/stream.py +++ b/stream.py @@ -445,7 +445,7 @@ def _set_cache_brute_(self, attr): #{ Configuration - if not has_perf_mod: + if has_perf_mod: _set_cache_ = _set_cache_brute_ else: _set_cache_ = _set_cache_too_slow_without_c From 21e2b5db748f18f70f16cd94ee9fd5adf16ce433 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 18 Oct 2010 21:31:41 +0200 Subject: [PATCH 0118/1392] Updated algorithm paper with the latest changes - thats it for now --- doc/source/algorithm.rst | 18 ++++++++---------- test/performance/test_pack.py | 4 ++-- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/doc/source/algorithm.rst b/doc/source/algorithm.rst index 55207b67b..4374cb820 100644 --- a/doc/source/algorithm.rst +++ b/doc/source/algorithm.rst @@ -58,10 +58,10 @@ GitDB's reverse delta aggregation algorithm =========================================== The idea of this algorithm is to merge all delta streams into one, which can then be applied in just one go. -In the current implementation, delta streams are parsed into DeltaChunks (->**DC**), which are kept in vectors. Each DC represents one copy-from-base operation, or one or multiple consecutive add-bytes operations. DeltaChunks know about their target offset in the target buffer, and their size. Their target offsets are consecutive, i.e. one chunk ends where the next one begins, regarding their logical extend in the target buffer. +In the current implementation, delta streams are parsed into DeltaChunks (->**DC**). Each DC represents one copy-from-base operation, or one or multiple consecutive add-bytes operations. DeltaChunks know about their target offset in the target buffer, and their size. Their target offsets are consecutive, i.e. one chunk ends where the next one begins, regarding their logical extend in the target buffer. Add-bytes DCs additional store their data to apply, copy-from-base DCs store the offset into the base buffer from which to copy bytes. -During processing, one starts with the latest (i.e. topmost) delta stream (->**TDS**), and iterates through its ancestor delta streams (->ADS) to merge them into the growing toplevel delta stream. +During processing, one starts with the latest (i.e. topmost) delta stream (->**TDS**), and iterates through its ancestor delta streams (->ADS) to merge them into the growing toplevel delta stream.. The merging works by following a set of rules: * Merge into the top-level delta from the youngest ancestor delta to the oldest one @@ -72,10 +72,9 @@ The merging works by following a set of rules: * Finish the merge once all ADS have been handled, or once the TDS only consists of add-byte DCs. The remaining copy-from-base DCs will copy from the original base buffer accordingly. -Applying the TDS is as straightforward as applying any other DS. The base buffer is required to be kept in memory. In the current implementation, a full-size target buffer is allocated to hold the result of applying the chunk information. +Applying the TDS is as straightforward as applying any other DS. The base buffer is required to be kept in memory. In the current implementation, a full-size target buffer is allocated to hold the result of applying the chunk information. Here it is already possible to stream the result, which is feasible only if the memory of the base buffer + the memory of the TDS are smaller than a full size target buffer. Streaming will always make sense if the peak resulting from having the base, target and TDS buffers in memory together is unaffordable. -The memory consumption during the TDS processing are the uncompressed delta-bytes, the parsed DS, as well as the TDS. Afterwards one requires an allocated base buffer, the target buffer, as well as the TDS. -It is clearly visible that the current implementation does not at all reduce memory consumption, but the opposite is true as the TDS can be large for large files. +The memory consumption during the TDS processing is only the condensed delta-bytes, for each ADS an additional index is required which costs 8 byte per DC. When applying the TDS, one requires an allocated base buffer too.The target buffer can be allocated, but may be a writer as well. Performance Results ------------------- @@ -83,17 +82,16 @@ The benchmarking context was the same as for the brute-force GitDB algorithm. Th The biggest performance bottleneck is the slicing of the parsed delta streams, where the program spends most of its time due to hundred thousands of calls. To get a more usable version of the algorithm, it was implemented in C, such that python must do no more than two calls to get all the work done. The first prepares the TDS, the second applies it, writing it into a target buffer. -The throughput reaches 16.7 MiB/s, which equals 1344 streams/s, which makes it 15 times faster than the pure python version, and amazingly even 1.5 times faster than the brute-force C implementation. As a comparison, cgit is able to stream about 20 MiB when controlling it through a pipe. GitDBs performance may still improve once pack access is reimplemented in C as well. +The throughput reaches 15.2 MiB/s, which equals 1221 streams/s, which makes it nearly 14 times faster than the pure python version, and amazingly even 1.35 times faster than the brute-force C implementation. As a comparison, cgit is able to stream about 20 MiB when controlling it through a pipe. GitDBs performance may still improve once pack access is reimplemented in C as well. -All this comes at a relatively high memory consumption.Additionally, with each new level being merged, not only are more DCs inserted, but the new chunks may get smaller as well. This can reach a point where one chunk only represents an individual byte, so the size of the data structure outweighs the logical chunk size by far. - -A 125 MB file took 3.1 seconds to unpack for instance, which is only 33% slower than the c implementation of the brute-force algorithm. +A 125 MB file took 2.5 seconds to unpack for instance, which is only 20% slower than the c implementation of the brute-force algorithm. Future work =========== -The current implementation of the reverse delta aggregation algorithm is already working well and fast, but leaves room for improvement in the realm of its memory consumption. One way to considerably reduce it would be to index the delta stream to determine bounds, instead of parsing it into a separate data structure Another very promising option is that streaming of delta data is indeed possible. Depending on the configuration of the copy-from-base operations, different optimizations could be applied to reduce the amount of memory required for the final processed delta stream. Some configurations may even allow it to stream data from the base buffer, instead of pre-loading it for random access. The ability to stream files at reduced memory costs would only be feasible for big files, and would have to be payed with extra pre-processing time. + +A very first and simple implementation could avoid memory peaks by streaming the TDS in conjunction with a base buffer, instead of writing everything into a fully allocated target buffer. diff --git a/test/performance/test_pack.py b/test/performance/test_pack.py index f9169ffb0..32890dcb3 100644 --- a/test/performance/test_pack.py +++ b/test/performance/test_pack.py @@ -13,7 +13,7 @@ class TestPackedDBPerformance(TestBigRepoR): - def test_pack_random_access(self): + def _test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # sha lookup @@ -61,7 +61,7 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) - def _disabled_test_correctness(self): + def test_correctness(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" From 2ddc5bad224d8f545ef3bb2ab3df98dfe063c5b6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 12 Nov 2010 18:49:01 +0100 Subject: [PATCH 0119/1392] Updated to latest revision of async to support new thread-shutdown functionality --- ext/async | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/async b/ext/async index 5992bb6c8..7298742e1 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 5992bb6c85973ed81c54c71fef42e2413cd29e88 +Subproject commit 7298742e1b7236f2a4e369f915722a6618cef736 From 2a048f43d89112ff1f78ee05b59a9663e981f63f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 18 Nov 2010 23:42:14 +0100 Subject: [PATCH 0120/1392] Changed name/id of async submodule to something that doesn't look like a path --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 45ddc0b4c..9adc6121b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "ext/async"] +[submodule "async"] path = ext/async url = git://gitorious.org/git-python/async.git From 479ac8efc6aa6c579cba48bbb87f85f1cd0654f5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 20 Nov 2010 22:51:56 +0100 Subject: [PATCH 0121/1392] bumped version to 0.5.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7e50e0fe7..6ebef0fa7 100755 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ def get_data_files(self): setup(cmdclass={'build_ext':build_ext_nofail}, name = "gitdb", - version = "0.5.1", + version = "0.5.2", description = "Git Object Database", author = "Sebastian Thiel", author_email = "byronimo@gmail.com", From dd4704095dbdc2770b9289b491bb166fa1d36a8d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 20 Nov 2010 22:59:24 +0100 Subject: [PATCH 0122/1392] Updated changelog for 0.5.2 --- doc/source/changes.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 84738ae05..7b8ebecc6 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -1,6 +1,12 @@ ######### Changelog ######### + +***** +0.5.2 +***** +* Improved performance of the c implementation, which now uses reverse-delta-aggregation to make a memory bound operation CPU bound. + ***** 0.5.1 ***** From 5d91a53372caa54cf7110cfa7fe1166956edc9c8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 21 Nov 2010 11:37:20 +0100 Subject: [PATCH 0123/1392] setup: added missing _delta_apply.c file to setup script, allowing the performance module to be compiled --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6ebef0fa7..c7d0bd81c 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def get_data_files(self): package_data={'gitdb' : ['AUTHORS', 'README'], 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':''}, - ext_modules=[Extension('gitdb._perf', ['_fun.c'])], + ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'])], license = "BSD License", requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', From 6edc28d6ab8b3f88673fd701b2c40104825a95df Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 21 Nov 2010 11:54:00 +0100 Subject: [PATCH 0124/1392] Added delta_apply.h file to make more native use of python's build system, which should hopefully fix the easy_install trouble --- MANIFEST.in | 2 ++ _delta_apply.c | 4 +++- _delta_apply.h | 6 ++++++ _fun.c | 2 +- setup.py | 3 ++- 5 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 _delta_apply.h diff --git a/MANIFEST.in b/MANIFEST.in index a01acc452..7693cabf8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,8 @@ include AUTHORS include README include _fun.c +include _delta_apply.c +include _delta_apply.h graft test diff --git a/_delta_apply.c b/_delta_apply.c index e99a803be..96ab30af9 100644 --- a/_delta_apply.c +++ b/_delta_apply.c @@ -1,10 +1,12 @@ -#include +#include "_delta_apply.h" #include #include #include #include #include + + typedef unsigned long long ull; typedef unsigned int uint; typedef unsigned char uchar; diff --git a/_delta_apply.h b/_delta_apply.h new file mode 100644 index 000000000..3e7e5f926 --- /dev/null +++ b/_delta_apply.h @@ -0,0 +1,6 @@ +#include + +static PyObject* connect_deltas(PyObject *self, PyObject *dstreams); +static PyObject* apply_delta(PyObject* self, PyObject* args); + +static PyTypeObject DeltaChunkListType; diff --git a/_fun.c b/_fun.c index befee4ec4..49970386f 100644 --- a/_fun.c +++ b/_fun.c @@ -1,5 +1,5 @@ #include -#include "_delta_apply.c" +#include "_delta_apply.h" static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) { diff --git a/setup.py b/setup.py index c7d0bd81c..3a95ac845 100755 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ def run(self): except Exception: print "Ignored failure when building extensions, pure python modules will be used instead" # END ignore errors + def get_data_files(self): """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, @@ -77,7 +78,7 @@ def get_data_files(self): package_data={'gitdb' : ['AUTHORS', 'README'], 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':''}, - ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'])], + ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'], include_dirs=['.'])], license = "BSD License", requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', From 1bc281d31b8d31fd4dcbcd9b441b5c7b2c1b0bb5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 21 Nov 2010 13:09:09 +0100 Subject: [PATCH 0125/1392] Added zip_safe flag to setup.py --- ext/async | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/async b/ext/async index 7298742e1..eccf3d63c 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit 7298742e1b7236f2a4e369f915722a6618cef736 +Subproject commit eccf3d63c655e7a184ba339d747c98e965c1a63b diff --git a/setup.py b/setup.py index 3a95ac845..d235c91df 100755 --- a/setup.py +++ b/setup.py @@ -80,6 +80,7 @@ def get_data_files(self): package_dir = {'gitdb':''}, ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'], include_dirs=['.'])], license = "BSD License", + zip_safe=False, requires=('async (>=0.6.1)',), install_requires='async >= 0.6.1', long_description = """GitDB is a pure-Python git object database""" From 9f977b8baaf9cbe9b38f3bdf4887cef5370b2229 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 25 Nov 2010 17:04:55 +0100 Subject: [PATCH 0126/1392] Switched async submodule to using github instead of gitorious --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 9adc6121b..42efc2ed6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "async"] path = ext/async - url = git://gitorious.org/git-python/async.git + url = git://github.com/Byron/async.git From c0d5448fcd5ed6427ea96be01ef2d3ee509f3924 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Nov 2010 23:09:53 +0100 Subject: [PATCH 0127/1392] Ajusted all links to point to new repository on github --- README => README.rst | 5 ++--- doc/source/intro.rst | 5 ++--- ext/async | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) rename README => README.rst (84%) diff --git a/README b/README.rst similarity index 84% rename from README rename to README.rst index 33a566d86..753eb701c 100644 --- a/README +++ b/README.rst @@ -15,8 +15,7 @@ SOURCE ====== The source is available in a git repository at gitorious and github: -git://gitorious.org/git-python/gitdb.git -git://github.com/Byron/gitdb.git +git://github.com/gitpython-developers/gitdb.git Once the clone is complete, please be sure to initialize the submodules using @@ -33,7 +32,7 @@ http://groups.google.com/group/git-python ISSUE TRACKER ============= -http://byronimo.lighthouseapp.com/projects/51787-gitpython +https://github.com/gitpython-developers/gitdb/issues LICENSE ======= diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 4d675cbc4..8fc0ec098 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -29,10 +29,9 @@ It is advised to have a look at the :ref:`Usage Guide ` for a br ================= Source Repository ================= -The latest source can be cloned using git from one of the following locations: +The latest source can be cloned using git from github: - * git://gitorious.org/git-python/gitdb.git - * git://github.com/Byron/gitdb.git + * git://github.com/gitpython-developers/gitdb.git License Information =================== diff --git a/ext/async b/ext/async index eccf3d63c..89790abd2 160000 --- a/ext/async +++ b/ext/async @@ -1 +1 @@ -Subproject commit eccf3d63c655e7a184ba339d747c98e965c1a63b +Subproject commit 89790abd29bb4be851095b2f2c4c624b896b6e20 From 9fbc59da76b15cecb1ee37a8e48617fab58a077c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Nov 2010 23:24:07 +0100 Subject: [PATCH 0128/1392] moved all relevant files into the gitdb folder. Submodule relinked to point to new github location, and moved as well --- .gitmodules | 4 ++-- __init__.py => gitdb/__init__.py | 0 _delta_apply.c => gitdb/_delta_apply.c | 0 _delta_apply.h => gitdb/_delta_apply.h | 0 _fun.c => gitdb/_fun.c | 0 base.py => gitdb/base.py | 0 {db => gitdb/db}/__init__.py | 0 {db => gitdb/db}/base.py | 0 {db => gitdb/db}/git.py | 0 {db => gitdb/db}/loose.py | 0 {db => gitdb/db}/mem.py | 0 {db => gitdb/db}/pack.py | 0 {db => gitdb/db}/ref.py | 0 exc.py => gitdb/exc.py | 0 {ext => gitdb/ext}/async | 0 fun.py => gitdb/fun.py | 0 pack.py => gitdb/pack.py | 0 stream.py => gitdb/stream.py | 0 {test => gitdb/test}/__init__.py | 0 {test => gitdb/test}/db/__init__.py | 0 {test => gitdb/test}/db/lib.py | 0 {test => gitdb/test}/db/test_git.py | 0 {test => gitdb/test}/db/test_loose.py | 0 {test => gitdb/test}/db/test_mem.py | 0 {test => gitdb/test}/db/test_pack.py | 0 {test => gitdb/test}/db/test_ref.py | 0 .../7b/b839852ed5e3a069966281bb08d50012fb309b | Bin ...ack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx | Bin ...ck-11fdfa9e156ab73caae3b6da867192221f2089c2.pack | Bin ...ack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx | Bin ...ck-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack | Bin ...ack-c0438c19fb16422b6bbcce24387b3264416d485b.idx | Bin ...ck-c0438c19fb16422b6bbcce24387b3264416d485b.pack | Bin {test => gitdb/test}/lib.py | 0 {test => gitdb/test}/performance/lib.py | 0 {test => gitdb/test}/performance/test_pack.py | 0 .../test}/performance/test_pack_streaming.py | 0 {test => gitdb/test}/performance/test_stream.py | 0 {test => gitdb/test}/test_base.py | 0 {test => gitdb/test}/test_example.py | 0 {test => gitdb/test}/test_pack.py | 0 {test => gitdb/test}/test_stream.py | 0 {test => gitdb/test}/test_util.py | 0 typ.py => gitdb/typ.py | 0 util.py => gitdb/util.py | 0 45 files changed, 2 insertions(+), 2 deletions(-) rename __init__.py => gitdb/__init__.py (100%) rename _delta_apply.c => gitdb/_delta_apply.c (100%) rename _delta_apply.h => gitdb/_delta_apply.h (100%) rename _fun.c => gitdb/_fun.c (100%) rename base.py => gitdb/base.py (100%) rename {db => gitdb/db}/__init__.py (100%) rename {db => gitdb/db}/base.py (100%) rename {db => gitdb/db}/git.py (100%) rename {db => gitdb/db}/loose.py (100%) rename {db => gitdb/db}/mem.py (100%) rename {db => gitdb/db}/pack.py (100%) rename {db => gitdb/db}/ref.py (100%) rename exc.py => gitdb/exc.py (100%) rename {ext => gitdb/ext}/async (100%) rename fun.py => gitdb/fun.py (100%) rename pack.py => gitdb/pack.py (100%) rename stream.py => gitdb/stream.py (100%) rename {test => gitdb/test}/__init__.py (100%) rename {test => gitdb/test}/db/__init__.py (100%) rename {test => gitdb/test}/db/lib.py (100%) rename {test => gitdb/test}/db/test_git.py (100%) rename {test => gitdb/test}/db/test_loose.py (100%) rename {test => gitdb/test}/db/test_mem.py (100%) rename {test => gitdb/test}/db/test_pack.py (100%) rename {test => gitdb/test}/db/test_ref.py (100%) rename {test => gitdb/test}/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b (100%) rename {test => gitdb/test}/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx (100%) rename {test => gitdb/test}/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack (100%) rename {test => gitdb/test}/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx (100%) rename {test => gitdb/test}/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack (100%) rename {test => gitdb/test}/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx (100%) rename {test => gitdb/test}/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack (100%) rename {test => gitdb/test}/lib.py (100%) rename {test => gitdb/test}/performance/lib.py (100%) rename {test => gitdb/test}/performance/test_pack.py (100%) rename {test => gitdb/test}/performance/test_pack_streaming.py (100%) rename {test => gitdb/test}/performance/test_stream.py (100%) rename {test => gitdb/test}/test_base.py (100%) rename {test => gitdb/test}/test_example.py (100%) rename {test => gitdb/test}/test_pack.py (100%) rename {test => gitdb/test}/test_stream.py (100%) rename {test => gitdb/test}/test_util.py (100%) rename typ.py => gitdb/typ.py (100%) rename util.py => gitdb/util.py (100%) diff --git a/.gitmodules b/.gitmodules index 42efc2ed6..9f06f7d07 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "async"] - path = ext/async - url = git://github.com/Byron/async.git + path = gitdb/ext/async + url = git://github.com/gitpython-developers/async.git diff --git a/__init__.py b/gitdb/__init__.py similarity index 100% rename from __init__.py rename to gitdb/__init__.py diff --git a/_delta_apply.c b/gitdb/_delta_apply.c similarity index 100% rename from _delta_apply.c rename to gitdb/_delta_apply.c diff --git a/_delta_apply.h b/gitdb/_delta_apply.h similarity index 100% rename from _delta_apply.h rename to gitdb/_delta_apply.h diff --git a/_fun.c b/gitdb/_fun.c similarity index 100% rename from _fun.c rename to gitdb/_fun.c diff --git a/base.py b/gitdb/base.py similarity index 100% rename from base.py rename to gitdb/base.py diff --git a/db/__init__.py b/gitdb/db/__init__.py similarity index 100% rename from db/__init__.py rename to gitdb/db/__init__.py diff --git a/db/base.py b/gitdb/db/base.py similarity index 100% rename from db/base.py rename to gitdb/db/base.py diff --git a/db/git.py b/gitdb/db/git.py similarity index 100% rename from db/git.py rename to gitdb/db/git.py diff --git a/db/loose.py b/gitdb/db/loose.py similarity index 100% rename from db/loose.py rename to gitdb/db/loose.py diff --git a/db/mem.py b/gitdb/db/mem.py similarity index 100% rename from db/mem.py rename to gitdb/db/mem.py diff --git a/db/pack.py b/gitdb/db/pack.py similarity index 100% rename from db/pack.py rename to gitdb/db/pack.py diff --git a/db/ref.py b/gitdb/db/ref.py similarity index 100% rename from db/ref.py rename to gitdb/db/ref.py diff --git a/exc.py b/gitdb/exc.py similarity index 100% rename from exc.py rename to gitdb/exc.py diff --git a/ext/async b/gitdb/ext/async similarity index 100% rename from ext/async rename to gitdb/ext/async diff --git a/fun.py b/gitdb/fun.py similarity index 100% rename from fun.py rename to gitdb/fun.py diff --git a/pack.py b/gitdb/pack.py similarity index 100% rename from pack.py rename to gitdb/pack.py diff --git a/stream.py b/gitdb/stream.py similarity index 100% rename from stream.py rename to gitdb/stream.py diff --git a/test/__init__.py b/gitdb/test/__init__.py similarity index 100% rename from test/__init__.py rename to gitdb/test/__init__.py diff --git a/test/db/__init__.py b/gitdb/test/db/__init__.py similarity index 100% rename from test/db/__init__.py rename to gitdb/test/db/__init__.py diff --git a/test/db/lib.py b/gitdb/test/db/lib.py similarity index 100% rename from test/db/lib.py rename to gitdb/test/db/lib.py diff --git a/test/db/test_git.py b/gitdb/test/db/test_git.py similarity index 100% rename from test/db/test_git.py rename to gitdb/test/db/test_git.py diff --git a/test/db/test_loose.py b/gitdb/test/db/test_loose.py similarity index 100% rename from test/db/test_loose.py rename to gitdb/test/db/test_loose.py diff --git a/test/db/test_mem.py b/gitdb/test/db/test_mem.py similarity index 100% rename from test/db/test_mem.py rename to gitdb/test/db/test_mem.py diff --git a/test/db/test_pack.py b/gitdb/test/db/test_pack.py similarity index 100% rename from test/db/test_pack.py rename to gitdb/test/db/test_pack.py diff --git a/test/db/test_ref.py b/gitdb/test/db/test_ref.py similarity index 100% rename from test/db/test_ref.py rename to gitdb/test/db/test_ref.py diff --git a/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b b/gitdb/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b similarity index 100% rename from test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b rename to gitdb/test/fixtures/objects/7b/b839852ed5e3a069966281bb08d50012fb309b diff --git a/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx b/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx similarity index 100% rename from test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx rename to gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx diff --git a/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack b/gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack similarity index 100% rename from test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack rename to gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack diff --git a/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx b/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx similarity index 100% rename from test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx rename to gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx diff --git a/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack b/gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack similarity index 100% rename from test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack rename to gitdb/test/fixtures/packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack diff --git a/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx b/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx similarity index 100% rename from test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx rename to gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx diff --git a/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack b/gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack similarity index 100% rename from test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack rename to gitdb/test/fixtures/packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack diff --git a/test/lib.py b/gitdb/test/lib.py similarity index 100% rename from test/lib.py rename to gitdb/test/lib.py diff --git a/test/performance/lib.py b/gitdb/test/performance/lib.py similarity index 100% rename from test/performance/lib.py rename to gitdb/test/performance/lib.py diff --git a/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py similarity index 100% rename from test/performance/test_pack.py rename to gitdb/test/performance/test_pack.py diff --git a/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py similarity index 100% rename from test/performance/test_pack_streaming.py rename to gitdb/test/performance/test_pack_streaming.py diff --git a/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py similarity index 100% rename from test/performance/test_stream.py rename to gitdb/test/performance/test_stream.py diff --git a/test/test_base.py b/gitdb/test/test_base.py similarity index 100% rename from test/test_base.py rename to gitdb/test/test_base.py diff --git a/test/test_example.py b/gitdb/test/test_example.py similarity index 100% rename from test/test_example.py rename to gitdb/test/test_example.py diff --git a/test/test_pack.py b/gitdb/test/test_pack.py similarity index 100% rename from test/test_pack.py rename to gitdb/test/test_pack.py diff --git a/test/test_stream.py b/gitdb/test/test_stream.py similarity index 100% rename from test/test_stream.py rename to gitdb/test/test_stream.py diff --git a/test/test_util.py b/gitdb/test/test_util.py similarity index 100% rename from test/test_util.py rename to gitdb/test/test_util.py diff --git a/typ.py b/gitdb/typ.py similarity index 100% rename from typ.py rename to gitdb/typ.py diff --git a/util.py b/gitdb/util.py similarity index 100% rename from util.py rename to gitdb/util.py From 0300d0d6c4af63257d094813ceaab2302906680c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Nov 2010 23:45:49 +0100 Subject: [PATCH 0129/1392] Fixed unittests --- doc/source/tutorial.rst | 2 +- gitdb/__init__.py | 8 +++++++- gitdb/test/db/test_git.py | 2 +- gitdb/test/db/test_pack.py | 3 +-- gitdb/test/db/test_ref.py | 2 +- gitdb/test/test_example.py | 2 +- 6 files changed, 12 insertions(+), 7 deletions(-) diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index cfe3fb284..55a737f6b 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -37,7 +37,7 @@ Both have two sets of methods, one of which allows interacting with single objec Acquiring information about an object from a database is easy if you have a SHA1 to refer to the object:: - ldb = LooseObjectDB(fixture_path("../../.git/objects")) + ldb = LooseObjectDB(fixture_path("../../../.git/objects")) for sha1 in ldb.sha_iter(): oinfo = ldb.info(sha1) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index d79788fc4..c8e77759e 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -6,7 +6,13 @@ #{ Initialization def _init_externals(): """Initialize external projects by putting them into the path""" - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext')) + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', 'async')) + + try: + import async + except ImportError: + raise ImportError("'async' could not be imported, assure it is located in your PYTHONPATH") + #END verify import #} END initialization diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index d2ae10bad..bcab1fc55 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -7,7 +7,7 @@ class TestGitDB(TestDBBase): def test_reading(self): - gdb = GitDB(fixture_path('../../.git/objects')) + gdb = GitDB(fixture_path('../../../.git/objects')) # we have packs and loose objects, alternates doesn't necessarily exist assert 1 < len(gdb.databases()) < 4 diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 0386b3f80..1d0cb96e7 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -15,8 +15,7 @@ def test_writing(self, path): pdb = PackedDB(path) # on demand, we init our pack cache - num_packs = 2 - assert len(pdb.entities()) == num_packs + num_packs = len(pdb.entities()) assert pdb._st_mtime != 0 # test pack directory changed: diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 9df25cef6..af75016b0 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -34,7 +34,7 @@ def test_writing(self, path): # setup alternate file # add two, one is invalid - own_repo_path = fixture_path('../../.git/objects') # use own repo + own_repo_path = fixture_path('../../../.git/objects') # use own repo self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) rdb.update_cache() assert len(rdb.databases()) == 1 diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 3fc3fcf5e..2d6096132 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -11,7 +11,7 @@ class TestExamples(TestBase): def test_base(self): - ldb = LooseObjectDB(fixture_path("../../.git/objects")) + ldb = LooseObjectDB(fixture_path("../../../.git/objects")) for sha1 in ldb.sha_iter(): oinfo = ldb.info(sha1) From 88d500edf2163be3b249ae288a06b725934560d9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 30 Nov 2010 23:57:16 +0100 Subject: [PATCH 0130/1392] setup and doc generation works once again --- MANIFEST.in | 8 ++++---- doc/source/conf.py | 2 +- gitdb/fun.py | 9 +++++---- gitdb/pack.py | 2 +- setup.py | 7 +++---- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 7693cabf8..b14aed9b3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,11 +4,11 @@ include CHANGES include AUTHORS include README -include _fun.c -include _delta_apply.c -include _delta_apply.h +include gitdb/_fun.c +include gitdb/_delta_apply.c +include gitdb/_delta_apply.h -graft test +prune gitdb/test global-exclude .git* global-exclude *.pyc diff --git a/doc/source/conf.py b/doc/source/conf.py index e10addb8d..28deb3106 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,7 +16,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.append(os.path.abspath('../../../')) +sys.path.append(os.path.abspath('../../')) # -- General configuration ----------------------------------------------------- diff --git a/gitdb/fun.py b/gitdb/fun.py index 0b14f82ac..fc4172040 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -44,8 +44,7 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'reverse_connect_deltas', - 'connect_deltas', 'DeltaChunkList') + 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList') #{ Structures @@ -492,8 +491,10 @@ def stream_copy(read, write, size, chunk_size): return dbw def connect_deltas(dstreams): - """Read the condensed delta chunk information from dstream and merge its information - into a list of existing delta chunks + """ + Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + :param dstreams: iterable of delta stream objects, the delta to be applied last comes first, then all its ancestors in order :return: DeltaChunkList, containing all operations to apply""" diff --git a/gitdb/pack.py b/gitdb/pack.py index 30da52c63..affcbe275 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -651,7 +651,7 @@ def is_valid_stream(self, sha, use_crc=False): :param use_crc: if True, the index' crc for the sha is used to determine :param sha: 20 byte sha1 of the object whose stream to verify - whether the compressed stream of the object is valid. If it is + whether the compressed stream of the object is valid. If it is a delta, this only verifies that the delta's data is valid, not the data of the actual undeltified object, as it depends on more than just this stream. diff --git a/setup.py b/setup.py index d235c91df..3c6617422 100755 --- a/setup.py +++ b/setup.py @@ -75,10 +75,9 @@ def get_data_files(self): author_email = "byronimo@gmail.com", url = "http://gitorious.org/git-python/gitdb", packages = ('gitdb', 'gitdb.db', 'gitdb.test', 'gitdb.test.db', 'gitdb.test.performance'), - package_data={'gitdb' : ['AUTHORS', 'README'], - 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, - package_dir = {'gitdb':''}, - ext_modules=[Extension('gitdb._perf', ['_fun.c', '_delta_apply.c'], include_dirs=['.'])], + package_data={ 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, + package_dir = {'gitdb':'gitdb'}, + ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, requires=('async (>=0.6.1)',), From 1fe2a9403fafa810af25062c7e6b20be8e6be480 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 1 Dec 2010 10:28:14 +0100 Subject: [PATCH 0131/1392] setup .gitmodules to use a trackin branch automatically --- .gitmodules | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitmodules b/.gitmodules index 9f06f7d07..3db4c676d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "async"] path = gitdb/ext/async url = git://github.com/gitpython-developers/async.git + branch = master From 6d315b8a92ae2cba936bd38e433592383f42cc10 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 23 Feb 2011 00:26:14 +0100 Subject: [PATCH 0132/1392] Added license information file --- LICENSE | 30 ++++++++++++++++++++++++++++++ gitdb/ext/async | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..be11e73c1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +Copyright (C) 2010, 2011 Sebastian Thiel and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of the GitDB project nor the names of +its contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/gitdb/ext/async b/gitdb/ext/async index 89790abd2..e91b46928 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 89790abd29bb4be851095b2f2c4c624b896b6e20 +Subproject commit e91b4692825e0121504262ba58ac0b2bcd09fea3 From df570f00f611073a20796128ca167474aa7826fc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 23 Feb 2011 00:43:31 +0100 Subject: [PATCH 0133/1392] preprended all modules with licensing information --- gitdb/__init__.py | 4 ++++ gitdb/base.py | 4 ++++ gitdb/db/__init__.py | 4 ++++ gitdb/db/base.py | 4 ++++ gitdb/db/git.py | 4 ++++ gitdb/db/loose.py | 4 ++++ gitdb/db/mem.py | 4 ++++ gitdb/db/pack.py | 4 ++++ gitdb/db/ref.py | 4 ++++ gitdb/exc.py | 4 ++++ gitdb/ext/async | 2 +- gitdb/fun.py | 4 ++++ gitdb/pack.py | 4 ++++ gitdb/stream.py | 4 ++++ gitdb/test/__init__.py | 4 ++++ gitdb/test/db/__init__.py | 4 ++++ gitdb/test/db/lib.py | 4 ++++ gitdb/test/db/test_git.py | 4 ++++ gitdb/test/db/test_loose.py | 4 ++++ gitdb/test/db/test_mem.py | 4 ++++ gitdb/test/db/test_pack.py | 4 ++++ gitdb/test/db/test_ref.py | 4 ++++ gitdb/test/lib.py | 4 ++++ gitdb/test/performance/lib.py | 4 ++++ gitdb/test/performance/test_pack.py | 4 ++++ gitdb/test/performance/test_pack_streaming.py | 4 ++++ gitdb/test/performance/test_stream.py | 4 ++++ gitdb/test/test_base.py | 4 ++++ gitdb/test/test_example.py | 4 ++++ gitdb/test/test_pack.py | 4 ++++ gitdb/test/test_stream.py | 4 ++++ gitdb/test/test_util.py | 4 ++++ gitdb/typ.py | 4 ++++ gitdb/util.py | 4 ++++ 34 files changed, 133 insertions(+), 1 deletion(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index c8e77759e..a551f37dc 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Initialize the object database module""" import sys diff --git a/gitdb/base.py b/gitdb/base.py index d0bcc0866..ff1062bf6 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with basic data structures - they are designed to be lightweight and fast""" from util import ( bin_to_hex, diff --git a/gitdb/db/__init__.py b/gitdb/db/__init__.py index 85a0a6874..e5935b7c2 100644 --- a/gitdb/db/__init__.py +++ b/gitdb/db/__init__.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import * from loose import * diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 1914dbbce..2189d4193 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains implementations of database retrieveing objects""" from gitdb.util import ( pool, diff --git a/gitdb/db/git.py b/gitdb/db/git.py index f0e63b15a..b8fc46aa0 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( CompoundDB, ObjectDBW, diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 521be44c2..6cd1cefd5 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( FileDBBase, ObjectDBR, diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index f361ab801..8012ad15e 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains the MemoryDatabase implementation""" from loose import LooseObjectDB from base import ( diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 0ec8a4e3b..eef3f712e 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing a database to deal with packs""" from base import ( FileDBBase, diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index c149c03d0..898984323 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( CompoundDB, ) diff --git a/gitdb/exc.py b/gitdb/exc.py index 012cdbc6b..96fa874e3 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with common exceptions""" from util import to_hex_sha diff --git a/gitdb/ext/async b/gitdb/ext/async index e91b46928..10310824c 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit e91b4692825e0121504262ba58ac0b2bcd09fea3 +Subproject commit 10310824c001deab8fea85b88ebda0696f964b3e diff --git a/gitdb/fun.py b/gitdb/fun.py index fc4172040..3e035dbf1 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains basic c-functions which usually contain performance critical code Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" diff --git a/gitdb/pack.py b/gitdb/pack.py index affcbe275..09e7defc5 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( BadObject, diff --git a/gitdb/stream.py b/gitdb/stream.py index 0d8972898..6c3b8d31f 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from cStringIO import StringIO import errno diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index 0dec7750f..760f531be 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php import gitdb.util diff --git a/gitdb/test/db/__init__.py b/gitdb/test/db/__init__.py index e69de29bb..8a681e428 100644 --- a/gitdb/test/db/__init__.py +++ b/gitdb/test/db/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 0080d919e..416c8c588 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Base classes for object db testing""" from gitdb.test.lib import ( with_rw_directory, diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index bcab1fc55..310116351 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.exc import BadObject from gitdb.db import GitDB diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 8e8a9bfc3..ee2d78d08 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import LooseObjectDB from gitdb.exc import BadObject diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index 4a9b7ee12..188cb0a93 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import ( MemoryDB, diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 1d0cb96e7..e8ba6f8fc 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import PackedDB from gitdb.test.lib import fixture_path diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index af75016b0..0d8eeebb3 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import ReferenceDB diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 3fb87d547..342234adc 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" from gitdb import ( OStream, diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index 45e0ca53f..761113d51 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains library functions""" import os from gitdb.test.lib import * diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index 32890dcb3..da952b17a 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance tests for object store""" from lib import ( TestBigRepoR diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 4d47cdfcc..22a62a39d 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Specific test for pack streams only""" from lib import ( TestBigRepoR diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 1afc1a1a0..f5f2e2e4d 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance data streaming performance""" from lib import TestBigRepoR from gitdb.db import * diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 740e50bcd..1b20faf87 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from lib import ( TestBase, diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 2d6096132..753177560 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with examples from the tutorial section of the docs""" from lib import * from gitdb import IStream diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 770a78bad..928f0cd97 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test everything about packs reading and writing""" from lib import ( TestBase, diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 948cbe766..523f77056 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from lib import ( TestBase, diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 6a389d27c..90f4156b9 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" import tempfile import os diff --git a/gitdb/typ.py b/gitdb/typ.py index 54a1f84be..e84dd2455 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing information about types known to the database""" #{ String types diff --git a/gitdb/util.py b/gitdb/util.py index 1ea182025..4bb3c7352 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -1,3 +1,7 @@ +# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors +# +# This module is part of GitDB and is released under +# the New BSD License: http://www.opensource.org/licenses/bsd-license.php import binascii import os import mmap From 3bcb30f1916239f1af25b4d6d8934c64bb47f8ea Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 11:06:36 +0200 Subject: [PATCH 0134/1392] Added qt creator project as it has advantages regarding the navigation over jEdit, although jedit has advantages regarding the syntax highlighting and whitespace visualization --- gitdb.pro | 48 +++++++++ gitdb.pro.user | 267 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 gitdb.pro create mode 100644 gitdb.pro.user diff --git a/gitdb.pro b/gitdb.pro new file mode 100644 index 000000000..a682615c4 --- /dev/null +++ b/gitdb.pro @@ -0,0 +1,48 @@ + +OTHER_FILES += \ + setup.py \ + README.rst \ + MANIFEST \ + Makefile \ + LICENSE \ + AUTHORS \ + gitdb/util.py \ + gitdb/typ.py \ + gitdb/stream.py \ + gitdb/pack.py \ + gitdb/__init__.py \ + gitdb/fun.py \ + gitdb/exc.py \ + gitdb/base.py \ + doc/source/tutorial.rst \ + doc/source/intro.rst \ + doc/source/index.rst \ + doc/source/conf.py \ + doc/source/changes.rst \ + doc/source/api.rst \ + doc/source/algorithm.rst \ + gitdb/db/ref.py \ + gitdb/db/pack.py \ + gitdb/db/mem.py \ + gitdb/db/loose.py \ + gitdb/db/__init__.py \ + gitdb/db/git.py \ + gitdb/db/base.py \ + gitdb/test/test_util.py \ + gitdb/test/test_stream.py \ + gitdb/test/test_pack.py \ + gitdb/test/test_example.py \ + gitdb/test/test_base.py \ + gitdb/test/lib.py \ + gitdb/test/__init__.py \ + gitdb/test/performance/test_stream.py \ + gitdb/test/performance/test_pack_streaming.py \ + gitdb/test/performance/test_pack.py \ + gitdb/test/performance/lib.py + +HEADERS += \ + gitdb/_delta_apply.h + +SOURCES += \ + gitdb/_fun.c \ + gitdb/_delta_apply.c diff --git a/gitdb.pro.user b/gitdb.pro.user new file mode 100644 index 000000000..398cb70a1 --- /dev/null +++ b/gitdb.pro.user @@ -0,0 +1,267 @@ + + + + ProjectExplorer.Project.ActiveTarget + 0 + + + ProjectExplorer.Project.EditorSettings + + Default + + + + ProjectExplorer.Project.Target.0 + + Desktop + + Qt4ProjectManager.Target.DesktopTarget + 0 + 0 + 1 + + + 0 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt in PATH Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 3 + 0 + false + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt in PATH Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 3 + 0 + true + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt 4.6.2 OpenSource Release + + Qt4ProjectManager.Qt4BuildConfiguration + 0 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 2 + 0 + true + + + + + qmake + + QtProjectManager.QMakeBuildStep + + + + Make + + Qt4ProjectManager.MakeStep + false + + + + 2 + Build + + ProjectExplorer.BuildSteps.Build + + + + Make + + Qt4ProjectManager.MakeStep + true + + clean + + + + 1 + Clean + + ProjectExplorer.BuildSteps.Clean + + 2 + false + + Qt 4.6.2 OpenSource Debug + + Qt4ProjectManager.Qt4BuildConfiguration + 2 + /home/byron/projects/git-python/git/ext/gitdb/gitdbpro-build-desktop + 2 + 0 + true + + 4 + + + 0 + Deploy + + ProjectExplorer.BuildSteps.Deploy + + 1 + No deployment + + ProjectExplorer.DefaultDeployConfiguration + + 1 + + gitdb + + Qt4ProjectManager.Qt4RunConfiguration + 2 + + gitdb.pro + false + false + + false + + 3768 + true + false + + + + /usr/bin/nosetests + -s + gitdb/test/test_pack.py + + 2 + python + false + + $BUILDDIR + Run python + test-pack + ProjectExplorer.CustomExecutableRunConfiguration + 3768 + true + false + + 2 + + + + ProjectExplorer.Project.TargetCount + 1 + + + ProjectExplorer.Project.Updater.EnvironmentId + {d38778e3-6b24-4419-8fc3-c8cc320f55e0} + + + ProjectExplorer.Project.Updater.FileVersion + 8 + + From 810d1e38315c6e886c1daef93670840b213ee78a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 11:08:26 +0200 Subject: [PATCH 0135/1392] Added stub for pack writing implementation which should work for pack streaming over a transport as well --- gitdb/fun.py | 10 +--------- gitdb/pack.py | 25 +++++++++++++++++++------ gitdb/test/test_pack.py | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index 3e035dbf1..34978cd97 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -411,15 +411,7 @@ def pack_object_header_info(data): size += (c & 0x7f) << s s += 7 # END character loop - - try: - return (type_id, size, i) - except KeyError: - # invalid object type - we could try to be smart now and decode part - # of the stream to get the info, problem is that we had trouble finding - # the exact start of the content stream - raise BadObjectType(type_id) - # END handle exceptions + return (type_id, size, i) def msb_size(data, offset=0): """ diff --git a/gitdb/pack.py b/gitdb/pack.py index 09e7defc5..335fe3c00 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -98,8 +98,7 @@ def pack_object_at(data, offset, as_stream): # REF DELTA elif type_id == REF_DELTA: total_rela_offset = data_rela_offset+20 - ref_sha = data[data_rela_offset:total_rela_offset] - delta_info = ref_sha + delta_info = data[data_rela_offset:total_rela_offset] # BASE OBJECT else: # assume its a base object @@ -561,11 +560,10 @@ def _sha_to_index(self, sha): def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" - indexfile = self._index + _sha = self._index.sha _object = self._object - for index in xrange(indexfile.size()): - sha = indexfile.sha(index) - yield _object(sha, as_stream, index) + for index in xrange(self._index.size()): + yield _object(_sha(index), as_stream, index) # END for each index def _object(self, sha, as_stream, index=-1): @@ -760,5 +758,20 @@ def collect_streams(self, sha): return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) + @classmethod + def create(cls, object_iter, pack_write, index_write=None): + """ + Create a new pack by putting all objects obtained by the object_iterator + into a pack which is written using the pack_write method. + The respective index is produced as well if index_write is not Non. + + :param object_iter: iterator yielding odb output objects + :param pack_write: function to receive strings to write into the pack stream + :param indx_write: if not None, the function writes the index file corresponding + to the pack. + :note: The destination of the write functions is up to the user. It could + be a socket, or a file for instance""" + + #} END interface diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 928f0cd97..8e98808c7 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -25,8 +25,12 @@ from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha -from itertools import izip +from itertools import izip, chain +from nose import SkipTest + import os +import sys +import tempfile #{ Utilities @@ -134,7 +138,9 @@ def test_pack(self): self._assert_pack_file(pack, version, size) # END for each pack to test - def test_pack_entity(self): + @with_rw_directory + def test_pack_entity(self, rw_dir): + pack_iterators = list(); for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): @@ -143,6 +149,7 @@ def test_pack_entity(self): entity = PackEntity(packfile) assert entity.pack().path() == packfile assert entity.index().path() == indexfile + pack_iterators.append(entity.stream_iter()) count = 0 for info, stream in izip(entity.info_iter(), entity.stream_iter()): @@ -174,9 +181,28 @@ def test_pack_entity(self): # END for each info, stream tuple assert count == size - # END for each entity + # END for each entity + + # pack writing - write all packs into one + # index path can be None + pack_path = tempfile.mktemp('', "pack", rw_dir) + index_path = tempfile.mktemp('', 'index', rw_dir) + for pp, ip in ((pack_path, )*2, (index_path, None)): + pfile = open(pp, 'wb') + ifile = None + if ip: + ifile = open(ip, 'wb') + #END handle ip + + PackEntity.create(chain(*pack_iterators), pfile, ifile) + assert os.path.getsize(pp) > 100 + if ip is not None: + assert os.path.getsize(ip) > 100 + #END verify files exist + #END for each packpath, indexpath pair + def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets # of course without really needing such a huge pack - pass + raise SkipTest() From e83210d99aaac5768827c448909fa04d63776e64 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 18:01:05 +0200 Subject: [PATCH 0136/1392] initial version of pack writing, which seems to work, but still needs some more testing and verification --- gitdb/exc.py | 3 + gitdb/fun.py | 20 +++- gitdb/pack.py | 196 ++++++++++++++++++++++++++++++++++++++-- gitdb/stream.py | 18 +++- gitdb/test/test_pack.py | 51 ++++++++--- 5 files changed, 266 insertions(+), 22 deletions(-) diff --git a/gitdb/exc.py b/gitdb/exc.py index 96fa874e3..e087047b4 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -17,6 +17,9 @@ class BadObject(ODBError): def __str__(self): return "BadObject: %s" % to_hex_sha(self.args[0]) + +class ParseError(ODBError): + """Thrown if the parsing of a file failed due to an invalid format""" class AmbiguousObjectName(ODBError): """Thrown if a possibly shortened name does not uniquely represent a single object diff --git a/gitdb/fun.py b/gitdb/fun.py index 34978cd97..5bbe8efc3 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -48,7 +48,7 @@ __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList') + 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') #{ Structures @@ -412,6 +412,24 @@ def pack_object_header_info(data): s += 7 # END character loop return (type_id, size, i) + +def create_pack_object_header(obj_type, obj_size): + """:return: string defining the pack header comprised of the object type + and its incompressed size in bytes + :parmam obj_type: pack type_id of the object + :param obj_size: uncompressed size in bytes of the following object stream""" + c = 0 # 1 byte + hdr = str() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr += chr(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + #END until size is consumed + hdr += chr(c) + return hdr def msb_size(data, offset=0): """ diff --git a/gitdb/pack.py b/gitdb/pack.py index 335fe3c00..6c32949d6 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -5,7 +5,8 @@ """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( BadObject, - UnsupportedOperation + UnsupportedOperation, + ParseError ) from util import ( zlib, @@ -15,6 +16,7 @@ ) from fun import ( + create_pack_object_header, pack_object_header_info, is_equal_canonical_sha, type_id_to_type_map, @@ -47,6 +49,7 @@ DeltaApplyReader, Sha1Writer, NullStream, + FlexibleSha1Writer ) from struct import ( @@ -54,6 +57,8 @@ unpack, ) +from binascii import crc32 + from itertools import izip import array import os @@ -119,10 +124,113 @@ def pack_object_at(data, offset, as_stream): return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) # END handle info # END handle stream - + +def write_stream_to_pack(read, write, zstream, want_crc=False): + """Copy a stream as read from read function, zip it, and write the result. + Count the number of written bytes and return it + :param want_crc: if True, the crc will be generated over the compressed data. + :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if want_crc + was false""" + br = 0 # bytes read + bw = 0 # bytes written + crc = 0 + + while True: + chunk = read(chunk_size) + br += len(chunk) + compressed = zstream.compress(chunk) + bw += len(compressed) + write(compressed) # cannot assume return value + + if want_crc: + crc = crc32(compressed, crc) + #END handle crc + + if len(chunk) != chunk_size: + break + #END copy loop + + compressed = zstream.flush() + bw += len(compressed) + write(compressed) + if want_crc: + crc = crc32(compressed, crc) + #END handle crc + + return (br, bw, crc) + + #} END utilities +class IndexWriter(object): + """Utility to cache index information, allowing to write all information later + in one go to the given stream + :note: currently only writes v2 indices""" + __slots__ = '_objs' + + def __init__(self): + self._objs = list() + + def append(self, binsha, crc, offset): + """Append one piece of object information""" + self._objs.append((binsha, crc, offset)) + + def write(self, pack_binsha, write): + """Write the index file using the given write method + :param pack_binsha: sha over the whole pack that we index""" + # sort for sha1 hash + self._objs.sort(key=lambda o: o[0]) + + sha_writer = FlexibleSha1Writer(write) + sha_write = sha_writer.write + sha_write(PackIndexFile.index_v2_signature) + sha_write(pack(">L", PackIndexFile.index_version_default)) + + # fanout + tmplist = list((0,)*256) # fanout or list with 64 bit offsets + for t in self._objs: + tmplist[ord(t[0][0])] += 1 + #END prepare fanout + + for i in xrange(255): + v = tmplist[i] + sha_write(pack('>L', v)) + tmplist[i+1] = v + #END write each fanout entry + sha_write(pack('>L', tmplist[255])) + + # sha1 ordered + # save calls, that is push them into c + sha_write(''.join(t[0] for t in self._objs)) + + # crc32 + for t in self._objs: + sha_write(pack('>L', t[1]&0xffffffff)) + #END for each crc + + tmplist = list() + # offset 32 + for t in self._objs: + ofs = t[2] + if ofs > 0x7fffffff: + tmplist.append(ofs) + ofs = 0x80000000 + len(tmplist)-1 + #END hande 64 bit offsets + sha_write(pack('>L', ofs&0xffffffff)) + #END for each offset + + # offset 64 + for ofs in tmplist: + sha_write(pack(">Q", ofs)) + #END for each offset + + # trailer + assert(len(pack_binsha) == 20) + sha_write(pack_binsha) + write(sha_writer.sha(as_hex=False)) + + class PackIndexFile(LazyMixin): """A pack index provides offsets into the corresponding pack, allowing to find @@ -135,6 +243,8 @@ class PackIndexFile(LazyMixin): # used in v2 indices _sha_list_offset = 8 + 1024 + index_v2_signature = '\377tOc' + index_version_default = 2 def __init__(self, indexpath): super(PackIndexFile, self).__init__() @@ -155,7 +265,7 @@ def _set_cache_(self, attr): # to access the fanout table or related properties # CHECK VERSION - self._version = (self._data[:4] == '\377tOc' and 2) or 1 + self._version = (self._data[:4] == self.index_v2_signature and 2) or 1 if self._version == 2: version_id = unpack_from(">L", self._data, 4)[0] assert version_id == self._version, "Unsupported index version: %i" % version_id @@ -383,6 +493,8 @@ class PackFile(LazyMixin): case""" __slots__ = ('_packpath', '_data', '_size', '_version') + pack_signature = 0x5041434b # 'PACK' + pack_version_default = 2 # offset into our data at which the first object starts first_object_offset = 3*4 # header bytes @@ -396,15 +508,19 @@ def _set_cache_(self, attr): self._data = file_contents_ro_filepath(self._packpath) # read the header information - type_id, self._version, self._size = unpack_from(">4sLL", self._data, 0) + type_id, self._version, self._size = unpack_from(">LLL", self._data, 0) # TODO: figure out whether we should better keep the lock, or maybe # add a .keep file instead ? else: # must be '_size' or '_version' # read header info - we do that just with a file stream - type_id, self._version, self._size = unpack(">4sLL", open(self._packpath).read(12)) + type_id, self._version, self._size = unpack(">LLL", open(self._packpath).read(12)) # END handle header + if type_id != self.pack_signature: + raise ParseError("Invalid pack signature: %i" % type_id) + #END assert type id + def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" data = self._data @@ -759,7 +875,8 @@ def collect_streams(self, sha): @classmethod - def create(cls, object_iter, pack_write, index_write=None): + def write_pack(cls, object_iter, pack_write, index_write=None, + object_count = None, zlib_compression = zlib.Z_BEST_SPEED): """ Create a new pack by putting all objects obtained by the object_iterator into a pack which is written using the pack_write method. @@ -769,9 +886,74 @@ def create(cls, object_iter, pack_write, index_write=None): :param pack_write: function to receive strings to write into the pack stream :param indx_write: if not None, the function writes the index file corresponding to the pack. + :param object_count: if you can provide the amount of objects in your iteration, + this would be the place to put it. Otherwise we have to pre-iterate and store + all items into a list to get the number, which uses more memory than necessary. + :param zlib_compression: the zlib compression level to use + :return: binary sha over all the contents of the pack :note: The destination of the write functions is up to the user. It could - be a socket, or a file for instance""" + be a socket, or a file for instance + :note: writes only undeltified objects""" + objs = object_iter + if not object_count: + if not isinstance(object_iter, (tuple, list)): + objs = list(object_iter) + #END handle list type + object_count = len(objs) + #END handle object + + pack_writer = FlexibleSha1Writer(pack_write) + pwrite = pack_writer.write + ofs = 0 # current offset into the pack file + index = None + wants_index = index_write is not None + + # write header + pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) + ofs += 12 + + if wants_index: + index = IndexWriter() + #END handle index header + + actual_count = 0 + for obj in objs: + actual_count += 1 + + # object header + hdr = create_pack_object_header(obj.type_id, obj.size) + pwrite(hdr) + + # data stream + zstream = zlib.compressobj(zlib_compression) + ostream = obj.stream + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, want_crc = index_write) + assert(br == obj.size) + if wants_index: + index.append(obj.binsha, crc, ofs) + #END handle index + + ofs += len(hdr) + bw + if actual_count == object_count: + break + #END abort once we are done + #END for each object + + if actual_count != object_count: + raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) + #END count assertion + + # write footer + binsha = pack_writer.sha(as_hex = False) + assert len(binsha) == 20 + pack_write(binsha) + ofs += len(binsha) # just for completeness ;) + + if wants_index: + index.write(binsha, index_write) + #END handle index + return binsha #} END interface diff --git a/gitdb/stream.py b/gitdb/stream.py index 6c3b8d31f..8010a0551 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -33,7 +33,9 @@ except ImportError: pass -__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader') +__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', + 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', + 'FDStream', 'NullStream') #{ RO Streams @@ -557,6 +559,20 @@ def sha(self, as_hex = False): #} END interface +class FlexibleSha1Writer(Sha1Writer): + """Writer producing a sha1 while passing on the written bytes to the given + write function""" + __slots__ = 'writer' + + def __init__(self, writer): + Sha1Writer.__init__(self) + self.writer = writer + + def write(self, data): + Sha1Writer.write(self, data) + self.writer(data) + + class ZippedStoreShaWriter(Sha1Writer): """Remembers everything someone writes to it and generates a sha""" __slots__ = ('buf', 'zip') diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 8e98808c7..c4d8df165 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -140,7 +140,7 @@ def test_pack(self): @with_rw_directory def test_pack_entity(self, rw_dir): - pack_iterators = list(); + pack_objs = list() for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): @@ -149,7 +149,7 @@ def test_pack_entity(self, rw_dir): entity = PackEntity(packfile) assert entity.pack().path() == packfile assert entity.index().path() == indexfile - pack_iterators.append(entity.stream_iter()) + pack_objs.extend(entity.stream_iter()) count = 0 for info, stream in izip(entity.info_iter(), entity.stream_iter()): @@ -182,24 +182,49 @@ def test_pack_entity(self, rw_dir): assert count == size # END for each entity - + # pack writing - write all packs into one # index path can be None pack_path = tempfile.mktemp('', "pack", rw_dir) index_path = tempfile.mktemp('', 'index', rw_dir) - for pp, ip in ((pack_path, )*2, (index_path, None)): - pfile = open(pp, 'wb') - ifile = None - if ip: - ifile = open(ip, 'wb') + iteration = 0 + for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): + pfile = open(ppath, 'wb') + iwrite = None + if ipath: + ifile = open(ipath, 'wb') + iwrite = ifile.write #END handle ip - PackEntity.create(chain(*pack_iterators), pfile, ifile) - assert os.path.getsize(pp) > 100 - if ip is not None: - assert os.path.getsize(ip) > 100 + # make sure we rewind the streams ... we work on the same objects over and over again + if iteration > 0: + for obj in pack_objs: + obj.stream.seek(0) + #END rewind streams + iteration += 1 + + binsha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) + pfile.close() + assert os.path.getsize(ppath) > 100 + + # verify pack + pf = PackFile(ppath) + assert pf.size() == len(pack_objs) + assert pf.version() == PackFile.pack_version_default + assert pf.checksum() == binsha + + # verify index + if ipath is not None: + assert os.path.getsize(ipath) > 100 + #END verify files exist - #END for each packpath, indexpath pair + + if ifile: + ifile.close() + #END handle index + #END for each packpath, indexpath pair + + # def test_pack_64(self): From 98a19ac1986b623277098263f01696827567c584 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 18:49:22 +0200 Subject: [PATCH 0137/1392] Implemented remainder of the test, and it already shows that something is wrong with my packs. Probably something stupid ;) --- gitdb/pack.py | 62 +++++++++++++++++++++++++++++++---------- gitdb/test/lib.py | 7 +++-- gitdb/test/test_pack.py | 34 +++++++++++++++------- 3 files changed, 76 insertions(+), 27 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index 6c32949d6..d90eeb991 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -12,6 +12,7 @@ zlib, LazyMixin, unpack_from, + bin_to_hex, file_contents_ro_filepath, ) @@ -60,6 +61,7 @@ from binascii import crc32 from itertools import izip +import tempfile import array import os import sys @@ -176,9 +178,10 @@ def append(self, binsha, crc, offset): """Append one piece of object information""" self._objs.append((binsha, crc, offset)) - def write(self, pack_binsha, write): + def write(self, pack_sha, write): """Write the index file using the given write method - :param pack_binsha: sha over the whole pack that we index""" + :param pack_sha: binary sha over the whole pack that we index + :return: sha1 binary sha over all index file contents""" # sort for sha1 hash self._objs.sort(key=lambda o: o[0]) @@ -192,11 +195,10 @@ def write(self, pack_binsha, write): for t in self._objs: tmplist[ord(t[0][0])] += 1 #END prepare fanout - for i in xrange(255): v = tmplist[i] sha_write(pack('>L', v)) - tmplist[i+1] = v + tmplist[i+1] += v #END write each fanout entry sha_write(pack('>L', tmplist[255])) @@ -226,9 +228,11 @@ def write(self, pack_binsha, write): #END for each offset # trailer - assert(len(pack_binsha) == 20) - sha_write(pack_binsha) - write(sha_writer.sha(as_hex=False)) + assert(len(pack_sha) == 20) + sha_write(pack_sha) + sha = sha_writer.sha(as_hex=False) + write(sha) + return sha @@ -767,7 +771,9 @@ def is_valid_stream(self, sha, use_crc=False): """ Verify that the stream at the given sha is valid. - :param use_crc: if True, the index' crc for the sha is used to determine + :param use_crc: if True, the index' crc is run over the compressed stream of + the object, which is much faster than checking the sha1. It is also + more prone to unnoticed corruption or manipulation. :param sha: 20 byte sha1 of the object whose stream to verify whether the compressed stream of the object is valid. If it is a delta, this only verifies that the delta's data is valid, not the @@ -890,7 +896,8 @@ def write_pack(cls, object_iter, pack_write, index_write=None, this would be the place to put it. Otherwise we have to pre-iterate and store all items into a list to get the number, which uses more memory than necessary. :param zlib_compression: the zlib compression level to use - :return: binary sha over all the contents of the pack + :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack + and over all contents of the index. If index_write was None, index_binsha will be None :note: The destination of the write functions is up to the user. It could be a socket, or a file for instance :note: writes only undeltified objects""" @@ -944,16 +951,41 @@ def write_pack(cls, object_iter, pack_write, index_write=None, #END count assertion # write footer - binsha = pack_writer.sha(as_hex = False) - assert len(binsha) == 20 - pack_write(binsha) - ofs += len(binsha) # just for completeness ;) + pack_sha = pack_writer.sha(as_hex = False) + assert len(pack_sha) == 20 + pack_write(pack_sha) + ofs += len(pack_sha) # just for completeness ;) + index_sha = None if wants_index: - index.write(binsha, index_write) + index_sha = index.write(pack_sha, index_write) #END handle index - return binsha + return pack_sha, index_sha + + @classmethod + def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + """Create a new on-disk entity comprised of a properly named pack file and a properly named + and corresponding index file. The pack contains all OStream objects contained in object iter. + :param base_dir: directory which is to contain the files + :return: PackEntity instance initialized with the new pack + :note: for more information on the other parameters see the write_pack method""" + pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) + index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) + pack_write = lambda d: os.write(pack_fd, d) + index_write = lambda d: os.write(index_fd, d) + + pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) + os.close(pack_fd) + os.close(index_fd) + + fmt = "pack-%s.%s" + new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) + new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) + os.rename(pack_path, new_pack_path) + os.rename(index_path, new_index_path) + + return cls(new_pack_path) #} END interface diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 342234adc..50645be65 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -42,19 +42,22 @@ def with_rw_directory(func): def wrapper(self): path = tempfile.mktemp(prefix=func.__name__) os.mkdir(path) + keep = False try: try: return func(self, path) except Exception: print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + keep = True raise finally: # Need to collect here to be sure all handles have been closed. It appears # a windows-only issue. In fact things should be deleted, as well as # memory maps closed, once objects go out of scope. For some reason # though this is not the case here unless we collect explicitly. - gc.collect() - shutil.rmtree(path) + if not keep: + gc.collect() + shutil.rmtree(path) # END handle exception # END wrapper diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index c4d8df165..e9c933e15 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -188,6 +188,10 @@ def test_pack_entity(self, rw_dir): pack_path = tempfile.mktemp('', "pack", rw_dir) index_path = tempfile.mktemp('', 'index', rw_dir) iteration = 0 + def rewind_streams(): + for obj in pack_objs: + obj.stream.seek(0) + #END utility for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): pfile = open(ppath, 'wb') iwrite = None @@ -198,12 +202,11 @@ def test_pack_entity(self, rw_dir): # make sure we rewind the streams ... we work on the same objects over and over again if iteration > 0: - for obj in pack_objs: - obj.stream.seek(0) + rewind_streams() #END rewind streams iteration += 1 - binsha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) + pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) pfile.close() assert os.path.getsize(ppath) > 100 @@ -211,20 +214,31 @@ def test_pack_entity(self, rw_dir): pf = PackFile(ppath) assert pf.size() == len(pack_objs) assert pf.version() == PackFile.pack_version_default - assert pf.checksum() == binsha + assert pf.checksum() == pack_sha # verify index if ipath is not None: + ifile.close() assert os.path.getsize(ipath) > 100 - + idx = PackIndexFile(ipath) + assert idx.version() == PackIndexFile.index_version_default + assert idx.packfile_checksum() == pack_sha + assert idx.indexfile_checksum() == index_sha + assert idx.size() == len(pack_objs) #END verify files exist - - if ifile: - ifile.close() - #END handle index #END for each packpath, indexpath pair - # + # verify the packs throughly + rewind_streams() + entity = PackEntity.create(pack_objs, rw_dir) + count = 0 + for info in entity.info_iter(): + count += 1 + for use_crc in reversed(range(2)): + assert entity.is_valid_stream(info.binsha, use_crc) + # END for each crc mode + #END for each info + assert count == len(pack_objs) def test_pack_64(self): From 184a776960efdc2a83eac571c9c046ffcee3e7c8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 20:27:44 +0200 Subject: [PATCH 0138/1392] crc needs to be done on the pack object header as well, of course --- gitdb/pack.py | 22 ++++++++++++++++++---- gitdb/test/test_pack.py | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index d90eeb991..7ae9786e6 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -127,15 +127,20 @@ def pack_object_at(data, offset, as_stream): # END handle info # END handle stream -def write_stream_to_pack(read, write, zstream, want_crc=False): +def write_stream_to_pack(read, write, zstream, base_crc=None): """Copy a stream as read from read function, zip it, and write the result. Count the number of written bytes and return it - :param want_crc: if True, the crc will be generated over the compressed data. - :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if want_crc + :param base_crc: if not None, the crc will be the base for all compressed data + we consecutively write and generate a crc32 from. If None, no crc will be generated + :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if base_crc was false""" br = 0 # bytes read bw = 0 # bytes written + want_crc = base_crc is not None crc = 0 + if want_crc: + crc = base_crc + #END initialize crc while True: chunk = read(chunk_size) @@ -651,6 +656,9 @@ def __init__(self, pack_or_index_path): def _set_cache_(self, attr): # currently this can only be _offset_map + # TODO: make this a simple sorted offset array which can be bisected + # to find the respective entry, from which we can take a +1 easily + # This might be slower, but should also be much lighter in memory ! offsets_sorted = sorted(self._index.offsets()) last_offset = len(self._pack.data()) - self._pack.footer_size assert offsets_sorted, "Cannot handle empty indices" @@ -926,15 +934,21 @@ def write_pack(cls, object_iter, pack_write, index_write=None, actual_count = 0 for obj in objs: actual_count += 1 + crc = 0 # object header hdr = create_pack_object_header(obj.type_id, obj.size) + if index_write: + crc = crc32(hdr) + else: + crc = None + #END handle crc pwrite(hdr) # data stream zstream = zlib.compressobj(zlib_compression) ostream = obj.stream - br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, want_crc = index_write) + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc = crc) assert(br == obj.size) if wants_index: index.append(obj.binsha, crc, ofs) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index e9c933e15..4a7f1caf2 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -234,7 +234,7 @@ def rewind_streams(): count = 0 for info in entity.info_iter(): count += 1 - for use_crc in reversed(range(2)): + for use_crc in range(2): assert entity.is_valid_stream(info.binsha, use_crc) # END for each crc mode #END for each info From 0c7a3ec9829caa6632afd3e46901be67c63ae7fa Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 31 Mar 2011 23:40:04 +0200 Subject: [PATCH 0139/1392] Fixed _perf module, which built, but didn't link dynamically. All the time, I think it never successfully imported, but its hard to believe this slipped by. Added performance test for pack-writing, which isn't really showing what I want as it currently read data from a densly compressed pack which takes most of the time in the nearly pure python implementation. Compared to c++, all the measured performance is just below anything I'd want to use. But we shouldn't forget this is just a test implementation, writing packs is quite simple actually, if you leave out the delta compression part and the delta logic --- gitdb/_delta_apply.c | 19 ++++---- gitdb/_delta_apply.h | 6 +-- gitdb/test/performance/test_pack_streaming.py | 43 +++++++++++++++++++ 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/gitdb/_delta_apply.c b/gitdb/_delta_apply.c index 96ab30af9..f03e7ea6d 100644 --- a/gitdb/_delta_apply.c +++ b/gitdb/_delta_apply.c @@ -1,4 +1,4 @@ -#include "_delta_apply.h" +#include <_delta_apply.h> #include #include #include @@ -463,7 +463,7 @@ void DIV_reset(DeltaInfoVector* vec) // Append one chunk to the end of the list, and return a pointer to it // It will not have been initialized ! -static inline +inline DeltaInfo* DIV_append(DeltaInfoVector* vec) { if (vec->size + 1 > vec->reserved_size){ @@ -703,7 +703,7 @@ typedef struct { } DeltaChunkList; -static + int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) { if(args && PySequence_Size(args) > 0){ @@ -715,20 +715,20 @@ int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) return 0; } -static + void DCL_dealloc(DeltaChunkList* self) { TSI_destroy(&(self->istream)); } -static + PyObject* DCL_py_rbound(DeltaChunkList* self) { return PyLong_FromUnsignedLongLong(self->istream.target_size); } // Write using a write function, taking remaining bytes from a base buffer -static + PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) { PyObject* pybuf = 0; @@ -769,13 +769,13 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) Py_RETURN_NONE; } -static PyMethodDef DCL_methods[] = { +PyMethodDef DCL_methods[] = { {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, {NULL} /* Sentinel */ }; -static PyTypeObject DeltaChunkListType = { +PyTypeObject DeltaChunkListType = { PyObject_HEAD_INIT(NULL) 0, /*ob_size*/ "DeltaChunkList", /*tp_name*/ @@ -897,7 +897,7 @@ uint compute_chunk_count(const uchar* data, const uchar* dend, bool read_header) return num_chunks; } -static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) +PyObject* connect_deltas(PyObject *self, PyObject *dstreams) { // obtain iterator PyObject* stream_iter = 0; @@ -1088,7 +1088,6 @@ static PyObject* connect_deltas(PyObject *self, PyObject *dstreams) // Write using a write function, taking remaining bytes from a base buffer // replaces the corresponding method in python -static PyObject* apply_delta(PyObject* self, PyObject* args) { PyObject* pybbuf = 0; diff --git a/gitdb/_delta_apply.h b/gitdb/_delta_apply.h index 3e7e5f926..1fcd53832 100644 --- a/gitdb/_delta_apply.h +++ b/gitdb/_delta_apply.h @@ -1,6 +1,6 @@ #include -static PyObject* connect_deltas(PyObject *self, PyObject *dstreams); -static PyObject* apply_delta(PyObject* self, PyObject* args); +extern PyObject* connect_deltas(PyObject *self, PyObject *dstreams); +extern PyObject* apply_delta(PyObject* self, PyObject* args); -static PyTypeObject DeltaChunkListType; +extern PyTypeObject DeltaChunkListType; diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 22a62a39d..795ed1e26 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -8,14 +8,57 @@ ) from gitdb.db.pack import PackedDB +from gitdb.stream import NullStream +from gitdb.pack import PackEntity import os import sys from time import time +from nose import SkipTest + +class CountedNullStream(NullStream): + __slots__ = '_bw' + def __init__(self): + self._bw = 0 + + def bytes_written(self): + return self._bw + + def write(self, d): + self._bw += NullStream.write(self, d) + class TestPackStreamingPerformance(TestBigRepoR): + def test_pack_writing(self): + # see how fast we can write a pack from object streams. + # This will not be fast, as we take time for decompressing the streams as well + ostream = CountedNullStream() + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + ni = 5000 + count = 0 + total_size = 0 + st = time() + objs = list() + for sha in pdb.sha_iter(): + count += 1 + objs.append(pdb.stream(sha)) + if count == ni: + break + #END gather objects for pack-writing + elapsed = time() - st + print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) + + st = time() + PackEntity.write_pack(objs, ostream.write) + elapsed = time() - st + total_kb = ostream.bytes_written() / 1000 + print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) + + def test_stream_reading(self): + raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # streaming only, meant for --with-profile runs From 03767cc17c6111c4c39cd4dde0517f5415515598 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 15:08:59 +0200 Subject: [PATCH 0140/1392] Initial setup for the testing framework; includes setup tools configuration and readme --- .gitignore | 6 +++++ Makefile | 37 +++++++++++++++++++++++++++++ README.rst | 50 +++++++++++++++++++++++++++++++++++++++ setup.py | 49 ++++++++++++++++++++++++++++++++++++++ smmap/__init__.py | 7 ++++++ smmap/mman.py | 3 +++ smmap/stream.py | 7 ++++++ smmap/test/__init__.py | 0 smmap/test/lib.py | 24 +++++++++++++++++++ smmap/test/test_mman.py | 7 ++++++ smmap/test/test_stream.py | 7 ++++++ 11 files changed, 197 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.rst create mode 100755 setup.py create mode 100644 smmap/__init__.py create mode 100644 smmap/mman.py create mode 100644 smmap/stream.py create mode 100644 smmap/test/__init__.py create mode 100644 smmap/test/lib.py create mode 100644 smmap/test/test_mman.py create mode 100644 smmap/test/test_stream.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..6cfb58df1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.pyc +build/ +.coverage +coverage +dist/ +MANIFEST diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..ca3051dd4 --- /dev/null +++ b/Makefile @@ -0,0 +1,37 @@ +.PHONY: build sdist cover test clean-files clean-docs doc all + +all: + $(info Possible targets:) + $(info doc) + $(info clean-docs) + $(info clean-files) + $(info clean) + $(info test) + $(info coverage) + $(info build) + $(info sdist) + +doc: + cd docs && make html + +clean-docs: + cd docs && make clean + +clean-files: + git clean -fx + +clean: clean-files clean-docs + +test: + nosetests + +coverage: + nosetests --with-coverage --cover-package=smmap + +build: + ./setup.py build + +sdist: + ./setup.py sdist + + diff --git a/README.rst b/README.rst new file mode 100644 index 000000000..08dc7111f --- /dev/null +++ b/README.rst @@ -0,0 +1,50 @@ +#################### +Sliding MMap (smmap) +#################### +A straight forward implementation of a slidinging memory map. +The idea is that every access to a file goes through a memory map manager, which will on demand map a region of a file and provide a string-like object for reading. + +When reading from it, you will have to check whether you are still within your window boundary, and possibly obtain a new window as required. + +The great benefit of this system is that you can use it to map files of any size even on 32 bit systems. Additionally it will be able to close unused windows right away to return system resources. If there are multiple clients for the same file and location, the same window will be reused as well. + +As there is a global management facility, you are also able to forcibly free all open handles which is handy on windows, which would otherwise prevent the deletion of the involved files. + +For convenience, a stream class is provided which hides the usage of the memory manager behind a simple stream interface. + +************ +LIMITATIONS +************ +The access is readonly by design. + +************ +REQUIREMENTS +************ +* Python 2.4 or higher + +******* +Install +******* +TODO + +****** +Source +****** +The source is available at git://github.com/Byron/smmap.git and can be cloned using:: + + git clone git://github.com/Byron/smmap.git + +************ +MAILING LIST +************ +http://groups.google.com/group/git-python + +************* +ISSUE TRACKER +************* +https://github.com/Byron/smmap/issues + +******* +LICENSE +******* +New BSD License diff --git a/setup.py b/setup.py new file mode 100755 index 000000000..e2bb622fd --- /dev/null +++ b/setup.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +import os +import codecs +try: + from setuptools import setup, find_packages +except ImportError: + from ez_setup import use_setuptools + use_setuptools() + from setuptools import setup, find_packages + +import smmap + +if os.path.exists("README.rst"): + long_description = codecs.open('README.rst', "r", "utf-8").read() +else: + long_description = "See http://github.com/nvie/smmap/tree/master" + +setup( + name="smmap", + version=smmap.__version__, + description="A pure git implementation of a sliding window memory map manager", + author=smmap.__author__, + author_email=smmap.__contact__, + url=smmap.__homepage__, + platforms=["any"], + license="BSD", + packages=find_packages(), + zip_safe=True, + classifiers=[ + # Picked from + # http://pypi.python.org/pypi?:action=list_classifiers + #"Development Status :: 1 - Planning", + #"Development Status :: 2 - Pre-Alpha", + #"Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", + #"Development Status :: 5 - Production/Stable", + #"Development Status :: 6 - Mature", + #"Development Status :: 7 - Inactive", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Windows", + "Operating System :: OSX", + "Programming Language :: Python", + ], + long_description=long_description, +) diff --git a/smmap/__init__.py b/smmap/__init__.py new file mode 100644 index 000000000..82cff638c --- /dev/null +++ b/smmap/__init__.py @@ -0,0 +1,7 @@ +"""Intialize the smmap package""" + +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/Byron/smmap" +version_info = (0, 8, 0) +__version__ = '.'.join(str(i) for i in version_info) diff --git a/smmap/mman.py b/smmap/mman.py new file mode 100644 index 000000000..dfa39db8c --- /dev/null +++ b/smmap/mman.py @@ -0,0 +1,3 @@ +"""Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" + +__all__ = [] diff --git a/smmap/stream.py b/smmap/stream.py new file mode 100644 index 000000000..bc5e568ea --- /dev/null +++ b/smmap/stream.py @@ -0,0 +1,7 @@ +"""Module with a simple stream implementation using the memory manager""" + +from mman import * + +__all__ = [] + + diff --git a/smmap/test/__init__.py b/smmap/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/smmap/test/lib.py b/smmap/test/lib.py new file mode 100644 index 000000000..450dd9dda --- /dev/null +++ b/smmap/test/lib.py @@ -0,0 +1,24 @@ +"""Provide base classes for the test system""" +from unittest import TestCase + +__all__ = ['TestBase'] + + +class TestBase(TestCase): + """Foundation used by all tests""" + + #{ Configuration + + #} END configuration + + #{ Overrides + @classmethod + def setUpAll(cls): + # nothing for now + pass + + #END overrides + + #{ Interface + + #} END interface diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py new file mode 100644 index 000000000..43aef7771 --- /dev/null +++ b/smmap/test/test_mman.py @@ -0,0 +1,7 @@ +from lib import TestBase + +from smmap.mman import * + +class TestMMan(TestBase): + def test_basics(self): + assert False diff --git a/smmap/test/test_stream.py b/smmap/test/test_stream.py new file mode 100644 index 000000000..fae928d9a --- /dev/null +++ b/smmap/test/test_stream.py @@ -0,0 +1,7 @@ +from lib import TestBase + +from smmap.stream import * + +class TestStream(TestBase): + def test_basics(self): + assert False From b75a09b997e278e5351b60bc17b738cf62594fe0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 15:19:45 +0200 Subject: [PATCH 0141/1392] Added docs framework --- doc/.gitignore | 2 + doc/Makefile | 89 +++++++++++++++++++ doc/make.bat | 113 ++++++++++++++++++++++++ doc/source/changes.rst | 9 ++ doc/source/conf.py | 194 +++++++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 23 +++++ 6 files changed, 430 insertions(+) create mode 100644 doc/.gitignore create mode 100644 doc/Makefile create mode 100644 doc/make.bat create mode 100644 doc/source/changes.rst create mode 100644 doc/source/conf.py create mode 100644 doc/source/index.rst diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 000000000..32060acdd --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1,2 @@ +build +*.version_info diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 000000000..675ec2094 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/smmap.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/smmap.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 000000000..6900a2a46 --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,113 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +set SPHINXBUILD=sphinx-build +set BUILDDIR=build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\smmap.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\smmap.ghc + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/doc/source/changes.rst b/doc/source/changes.rst new file mode 100644 index 000000000..ee17e0af0 --- /dev/null +++ b/doc/source/changes.rst @@ -0,0 +1,9 @@ +######### +Changelog +######### + +********** +v0.8.0 +********** + +- Initial Release diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 000000000..a0dac1166 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +# +# smmap documentation build configuration file, created by +# sphinx-quickstart on Wed Jun 8 15:14:25 2011. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'smmap' +copyright = u'2011, Sebastian Thiel' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.8.0' +# The full version, including alpha/beta/rc tags. +release = '0.8.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['.static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'smmapdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'smmap.tex', u'smmap Documentation', + u'Sebastian Thiel', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 000000000..cb03044e0 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,23 @@ +.. smmap documentation master file, created by + sphinx-quickstart on Wed Jun 8 15:14:25 2011. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to smmap's documentation! +================================= +**smmap** is a pure python implementation of a sliding memory map to help unifying memory mapped access on 32 and 64 bit systems and to help managing resources more efficiently. + +Contents: + +.. toctree:: + :maxdepth: 2 + + changes + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + From 2156e9ab02ea27289e6b26c11b024685b3816935 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 19:15:41 +0200 Subject: [PATCH 0142/1392] Implemented Window including test. Started Region implementation as well as test, but noticed that a critical feature, the mmap's offset, doesn't exist prior to python 2.6. Its total crap, so is python --- README.rst | 5 +- smmap/mman.py | 110 +++++++++++++++++++++++++++++++++++++++- smmap/test/lib.py | 43 +++++++++++++++- smmap/test/test_mman.py | 67 +++++++++++++++++++++++- 4 files changed, 219 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 08dc7111f..4c22eed2b 100644 --- a/README.rst +++ b/README.rst @@ -15,12 +15,13 @@ For convenience, a stream class is provided which hides the usage of the memory ************ LIMITATIONS ************ -The access is readonly by design. +* The access is readonly by design. +* In python below 2.6, memory maps will be created in compatability mode which works, but creates inefficient memory maps as they always start at offset 0. ************ REQUIREMENTS ************ -* Python 2.4 or higher +* runs Python 2.4 or higher, but needs Python 2.6 or higher to run properly as it needs the offset parameter of the mmap.mmap function. ******* Install diff --git a/smmap/mman.py b/smmap/mman.py index dfa39db8c..004987a8f 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,3 +1,111 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" -__all__ = [] +__all__ = ["MappedMemoryManager"] + +import os +import mmap + +from mmap import PAGESIZE + +#{ Utilities + +def align_to_page(num, round_up): + """Align the given integer number to the closest page offset, which usually is 4096 bytes. + :param round_up: if True, the next higher multiple of page size is used, otherwise + the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) + :return: num rounded to closest page""" + res = (num / PAGESIZE) * PAGESIZE; + if round_up and (res != num): + res += PAGESIZE; + #END handle size + return res; + +#}END utilities + +class Window(object): + """Utility type which is used to snap windows towards each other, and to adjust their size""" + __slots__ = ( + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) + + def __init__(self, offset, size): + self.ofs = offset + self.size = size + + def __repr__(self): + return "Window(%i, %i)" % (self.ofs, self.size) + + @classmethod + def from_region(cls, region): + """:return: new window from a region""" + return cls(region.ofs_begin(), region.size()) + + def ofs_end(self): + return self.ofs + self.size + + def align(self): + self.ofs = align_to_page(self.ofs, 0) + self.size = align_to_page(self.size, 1) + + def extend_left_to(self, window, max_size): + """Adjust the offset to start where the given window on our left ends if possible, + but don't make yourself larger than max_size. + The resize will assure that the new window still contains the old window area""" + rofs = self.ofs - window.ofs_end() + nsize = rofs + self.size + rofs -= nsize - min(nsize, max_size) + self.ofs = self.ofs - rofs + self.size += rofs + + def extend_right_to(self, window, max_size): + """Adjust the size to make our window end where the right window begins, but don't + get larger than max_size""" + self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) + + +class Region(object): + """Defines a mapped region of memory, aligned to pagesizes + :note: deallocates used region automatically on destruction""" + __slots__ = ( + '_b' , # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_nc', # number of clients using this region + '_uc' # total amount of usages + ) + + + def __init__(self, path, ofs, size): + """Initialize a region, allocate the memory map + :param path: path to the file to map + :param ofs: **aligned** offset into the file to be mapped + :param size: if size is larger then the file on disk, the whole file will be + allocated the the size automatically adjusted + :raise Exception: if no memory can be allocated""" + self._b = ofs + self._nc = 0 + self._uc = 0 + + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + try: + self._mf = mmap.mmap(fd, size, access=mmap.ACCESS_READ, offset=ofs) + finally: + os.close(fd) + #END close file handle + + +class MappedMemoryManager(object): + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily + obtain additional regions assuring there is no overlap. + Once a certain memory limit is reached globally, or if there cannot be more open file handles + which result from each mmap call, the least recently used, and currently unused mapped regions + are unloaded automatically. + + :note: currently not thread-safe ! + :note: in the current implementation, we will automatically unload windows if we either cannot + create more memory maps (as the open file handles limit is hit) or if we have allocated more than + a safe amount of memory already, which would possibly cause memory allocations to fail as our address + space is full.""" + + __slots__ = tuple() + diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 450dd9dda..0605313d3 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -1,9 +1,50 @@ """Provide base classes for the test system""" from unittest import TestCase +import os +import tempfile -__all__ = ['TestBase'] +__all__ = ['TestBase', 'FileCreator'] +#{ Utilities + +class FileCreator(object): + """A instance which creates a temporary file with a prefix and a given size + and provides this info to the user. + Once it gets deleted, it will remove the temporary file as well.""" + __slots__ = ("_size", "_path") + + def __init__(self, size, prefix=''): + assert size, "Require size to be larger 0" + + self._path = tempfile.mktemp(prefix=prefix) + self._size = size + + fp = open(self._path, "wb") + fp.seek(size-1) + fp.write('1') + fp.close() + + assert os.path.getsize(self.path) == size + + def __del__(self): + try: + os.remove(self.path) + except OSError: + pass + #END exception handling + + + @property + def path(self): + return self._path + + @property + def size(self): + return self._size + +#} END utilities + class TestBase(TestCase): """Foundation used by all tests""" diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 43aef7771..faee8c9cf 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,70 @@ -from lib import TestBase +from lib import TestBase, FileCreator from smmap.mman import * +from smmap.mman import Region +from smmap.mman import Window + +import sys +import mmap class TestMMan(TestBase): + + _window_test_size = 1000 * 1000 * 8 + 5195 + + def test_window(self): + wl = Window(0, 1) # left + wc = Window(1, 1) # center + wc2 = Window(10, 5) # another center + wr = Window(8000, 50) # right + + assert wl.ofs_end() == 1 + assert wc.ofs_end() == 2 + assert wr.ofs_end() == 8050 + + # extension does nothing if already in place + maxsize = 100 + wc.extend_left_to(wl, maxsize) + assert wc.ofs == 1 and wc.size == 1 + wl.extend_right_to(wc, maxsize) + wl.extend_right_to(wc, maxsize) + assert wl.ofs == 0 and wl.size == 1 + + # an actual left extension + pofs_end = wc2.ofs_end() + wc2.extend_left_to(wc, maxsize) + assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() + + + # respects maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + + # without maxsize + wc.extend_right_to(wr, sys.maxint) + assert wc.ofs_end() == wr.ofs and wc.ofs == 1 + + # extend left + wr.extend_left_to(wc2, maxsize) + wr.extend_left_to(wc2, maxsize) + assert wr.size == maxsize + + wr.extend_left_to(wc2, sys.maxint) + assert wr.ofs == wc2.ofs_end() + + wc.align() + assert wc.ofs == 0 and wc.size == mmap.PAGESIZE*2 + + + def test_region(self): + fc = FileCreator(self._window_test_size, "window_test") + rfull = Region(fc.path, 0, fc.size) + + + + Window.from_region # todo + pass + def test_basics(self): - assert False + pass From 66a78db0127ed84e22cda5aed2009955765959e4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 20:26:51 +0200 Subject: [PATCH 0143/1392] Implemented MappedRegion type including test --- smmap/mman.py | 73 ++++++++++++++++++++++++++++++++++++++--- smmap/test/test_mman.py | 23 ++++++++++--- 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 004987a8f..c5a388750 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -3,6 +3,7 @@ __all__ = ["MappedMemoryManager"] import os +import sys import mmap from mmap import PAGESIZE @@ -64,16 +65,22 @@ def extend_right_to(self, window, max_size): self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) -class Region(object): +class MappedRegion(object): """Defines a mapped region of memory, aligned to pagesizes :note: deallocates used region automatically on destruction""" - __slots__ = ( + __slots__ = [ '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_nc', # number of clients using this region - '_uc' # total amount of usages - ) + '_uc', # total amount of usages + '_ms' # actual size of the mapping + ] + _need_compat_layer = sys.version_info[1] < 6 + if _need_compat_layer: + __slots__.append('_mfb') # mapped memory buffer to provide offset + #END handle additional slot + def __init__(self, path, ofs, size): """Initialize a region, allocate the memory map @@ -88,10 +95,66 @@ def __init__(self, path, ofs, size): fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) try: - self._mf = mmap.mmap(fd, size, access=mmap.ACCESS_READ, offset=ofs) + kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) + corrected_size = size + if self._need_compat_layer: + del(kwargs['offset']) + corrected_size += ofs + # END handle python not supporting offset ! Arg + + # have to correct size, otherwise (instead of the c version) it will + # bark that the size is too large ... many extra file accesses because + # if this ... argh ! + self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size, corrected_size), **kwargs) + + print len(self._mf) + if self._need_compat_layer: + self._mfb = buffer(self._mf, ofs, size) + #END handle buffer wrapping finally: os.close(fd) #END close file handle + + def ofs_begin(self): + """:return: absolute byte offset to the first byte of the mapping""" + return self._b + + def size(self): + """:return: total size of the mapped region in bytes""" + return len(self._mf) + + def ofs_end(self): + """:return: Absolute offset to one byte beyond the mapping into the file""" + return self._b + self.size() + + def includes_ofs(self, ofs): + """:return: True if the given offset can be read in our mapped region""" + return (ofs >= self.ofs_begin()) and (ofs <= self.ofs_end()) + + def client_count(self): + """:return: number of clients currently using this region""" + return self._nc + + def adjust_client_count(self, ofs): + """Adjust the client count by the given positive or negative offset""" + self._nc += ofs + + def usage_count(self): + """:return: amount of usages so far""" + return self._uc + + def adjust_usage_count(self, ofs): + """Adjust the usage count by the given positive or negative offset""" + self._uc += ofs + + # re-define all methods which need offset adjustments in compatibility mode + if _need_compat_layer: + def size(self): + return len(self._mf) - self._b + + def ofs_end(self): + return len(self._mf) + #END handle compat layer class MappedMemoryManager(object): diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index faee8c9cf..482ef923c 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,7 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import Region +from smmap.mman import MappedRegion from smmap.mman import Window import sys @@ -59,12 +59,27 @@ def test_window(self): def test_region(self): fc = FileCreator(self._window_test_size, "window_test") - rfull = Region(fc.path, 0, fc.size) + half_size = fc.size / 2 + rofs = 4000 + rfull = MappedRegion(fc.path, 0, fc.size) + rhalfofs = MappedRegion(fc.path, rofs, fc.size) + rhalfsize = MappedRegion(fc.path, 0, half_size) + # offsets + assert rfull.ofs_begin() == 0 and rfull.size() == fc.size + assert rfull.ofs_end() == fc.size # if this method works, it works always + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs + assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size + + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) + assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + + # window constructor + w = Window.from_region(rfull) + assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() - Window.from_region # todo - pass def test_basics(self): pass From 0bf73877f860a86d9cf601154e9a9f76292e63c9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 21:00:33 +0200 Subject: [PATCH 0144/1392] Fixed bug in test case as it didn't properly align its offset to a page --- smmap/mman.py | 20 +++++++++++++++----- smmap/test/test_mman.py | 18 +++++++++++++++--- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index c5a388750..27904f5d0 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -7,6 +7,7 @@ import mmap from mmap import PAGESIZE +from sys import getrefcount #{ Utilities @@ -71,7 +72,6 @@ class MappedRegion(object): __slots__ = [ '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) - '_nc', # number of clients using this region '_uc', # total amount of usages '_ms' # actual size of the mapping ] @@ -90,24 +90,24 @@ def __init__(self, path, ofs, size): allocated the the size automatically adjusted :raise Exception: if no memory can be allocated""" self._b = ofs - self._nc = 0 self._uc = 0 fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) try: kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) corrected_size = size + sizeofs = ofs if self._need_compat_layer: del(kwargs['offset']) corrected_size += ofs + sizeofs = 0 # END handle python not supporting offset ! Arg # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! - self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size, corrected_size), **kwargs) + self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size - sizeofs), **kwargs) - print len(self._mf) if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, size) #END handle buffer wrapping @@ -133,7 +133,8 @@ def includes_ofs(self, ofs): def client_count(self): """:return: number of clients currently using this region""" - return self._nc + # -1: self on stack, -1 self in this method, -1 self in getrefcount + return getrefcount(self)-3 def adjust_client_count(self, ofs): """Adjust the client count by the given positive or negative offset""" @@ -157,6 +158,15 @@ def ofs_end(self): #END handle compat layer +class Cursor(object): + """Pointer into the mapped region of the memory manager, keeping the current window + alive until it is destroyed""" + + +class MappedRegionList(list): + """List of MappedRegion instances with specific functionality""" + + class MappedMemoryManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 482ef923c..00c9d90e8 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,8 +1,11 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import MappedRegion +from smmap.mman import align_to_page from smmap.mman import Window +from smmap.mman import MappedRegion +from smmap.mman import MappedRegionList +from smmap.mman import Cursor import sys import mmap @@ -56,11 +59,10 @@ def test_window(self): wc.align() assert wc.ofs == 0 and wc.size == mmap.PAGESIZE*2 - def test_region(self): fc = FileCreator(self._window_test_size, "window_test") half_size = fc.size / 2 - rofs = 4000 + rofs = align_to_page(4200, False) rfull = MappedRegion(fc.path, 0, fc.size) rhalfofs = MappedRegion(fc.path, rofs, fc.size) rhalfsize = MappedRegion(fc.path, 0, half_size) @@ -76,10 +78,20 @@ def test_region(self): assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + # auto-refcount + assert rfull.client_count() == 1 + rfull2 = rfull + assert rfull.client_count() == 2 + # window constructor w = Window.from_region(rfull) assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() + def test_region_list(self): + pass + + def test_cursor(self): + pass def test_basics(self): pass From 25e50356ab7c5392cece8132a8ee5b99c1789734 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 21:25:07 +0200 Subject: [PATCH 0145/1392] Added rather trivial implementation for the region list, including test --- smmap/mman.py | 30 ++++++++++++++++++++++++++---- smmap/test/test_mman.py | 7 ++++++- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 27904f5d0..e0b4a5e64 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -158,15 +158,37 @@ def ofs_end(self): #END handle compat layer +class MappedRegionList(list): + """List of MappedRegion instances associating a path with a list of regions.""" + __slots__ = ( + '_path', # path which is mapped by all our regions + '_file_size' # total size of the file we map + ) + + def __new__(cls, path): + return super(MappedRegionList, cls).__new__(cls) + + def __init__(self, path): + self._path = path + self._file_size = None + + def path(self): + """:return: path to file whose regions we manage""" + return self._path + + def file_size(self): + """:return: size of file we manager""" + if self._file_size is None: + self._file_size = os.stat(self._path).st_size + #END update file size + return self._file_size + + class Cursor(object): """Pointer into the mapped region of the memory manager, keeping the current window alive until it is destroyed""" - -class MappedRegionList(list): - """List of MappedRegion instances with specific functionality""" - class MappedMemoryManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 00c9d90e8..2027d9004 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -88,7 +88,12 @@ def test_region(self): assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() def test_region_list(self): - pass + fc = FileCreator(100, "sample_file") + ml = MappedRegionList(fc.path) + + assert len(ml) == 0 + assert ml.path() == fc.path + assert ml.file_size() == fc.size def test_cursor(self): pass From cab0e3d6d995b7814e09157086d07c3ca2c901d4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 22:36:11 +0200 Subject: [PATCH 0146/1392] Moved all utility types into their own module --- smmap/mman.py | 204 ++++++---------------------------------- smmap/test/lib.py | 2 +- smmap/test/test_mman.py | 93 +----------------- smmap/test/test_util.py | 89 ++++++++++++++++++ smmap/util.py | 189 +++++++++++++++++++++++++++++++++++++ 5 files changed, 309 insertions(+), 268 deletions(-) create mode 100644 smmap/test/test_util.py create mode 100644 smmap/util.py diff --git a/smmap/mman.py b/smmap/mman.py index e0b4a5e64..0ecc1aad1 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,192 +1,46 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" -__all__ = ["MappedMemoryManager"] +__all__ = ["MappedMemoryManager", "MemoryCursor"] -import os -import sys -import mmap - -from mmap import PAGESIZE -from sys import getrefcount - -#{ Utilities - -def align_to_page(num, round_up): - """Align the given integer number to the closest page offset, which usually is 4096 bytes. - :param round_up: if True, the next higher multiple of page size is used, otherwise - the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) - :return: num rounded to closest page""" - res = (num / PAGESIZE) * PAGESIZE; - if round_up and (res != num): - res += PAGESIZE; - #END handle size - return res; - -#}END utilities - -class Window(object): - """Utility type which is used to snap windows towards each other, and to adjust their size""" - __slots__ = ( - 'ofs', # offset into the file in bytes - 'size' # size of the window in bytes +from util import ( + MemoryWindow, + MappedRegion, + MappedRegionList, ) - def __init__(self, offset, size): - self.ofs = offset - self.size = size - - def __repr__(self): - return "Window(%i, %i)" % (self.ofs, self.size) - - @classmethod - def from_region(cls, region): - """:return: new window from a region""" - return cls(region.ofs_begin(), region.size()) - - def ofs_end(self): - return self.ofs + self.size - def align(self): - self.ofs = align_to_page(self.ofs, 0) - self.size = align_to_page(self.size, 1) - - def extend_left_to(self, window, max_size): - """Adjust the offset to start where the given window on our left ends if possible, - but don't make yourself larger than max_size. - The resize will assure that the new window still contains the old window area""" - rofs = self.ofs - window.ofs_end() - nsize = rofs + self.size - rofs -= nsize - min(nsize, max_size) - self.ofs = self.ofs - rofs - self.size += rofs - - def extend_right_to(self, window, max_size): - """Adjust the size to make our window end where the right window begins, but don't - get larger than max_size""" - self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) - - -class MappedRegion(object): - """Defines a mapped region of memory, aligned to pagesizes - :note: deallocates used region automatically on destruction""" - __slots__ = [ - '_b' , # beginning of mapping - '_mf', # mapped memory chunk (as returned by mmap) - '_uc', # total amount of usages - '_ms' # actual size of the mapping - ] - _need_compat_layer = sys.version_info[1] < 6 - - if _need_compat_layer: - __slots__.append('_mfb') # mapped memory buffer to provide offset - #END handle additional slot - +class MemoryCursor(object): + """Pointer into the mapped region of the memory manager, keeping the current window + alive until it is destroyed""" + __slots__ = ( + '_manager', # the manger keeping all file regions + '_regions', # a regions list with regions for our file + '_region', # WEAK REF to our current region + '_ofs', # relative offset from the actually mapped area to our start area + '_size' # maximum size we should provide + ) - def __init__(self, path, ofs, size): - """Initialize a region, allocate the memory map - :param path: path to the file to map - :param ofs: **aligned** offset into the file to be mapped - :param size: if size is larger then the file on disk, the whole file will be - allocated the the size automatically adjusted - :raise Exception: if no memory can be allocated""" - self._b = ofs - self._uc = 0 - - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) - try: - kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) - corrected_size = size - sizeofs = ofs - if self._need_compat_layer: - del(kwargs['offset']) - corrected_size += ofs - sizeofs = 0 - # END handle python not supporting offset ! Arg - - # have to correct size, otherwise (instead of the c version) it will - # bark that the size is too large ... many extra file accesses because - # if this ... argh ! - self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size - sizeofs), **kwargs) - - if self._need_compat_layer: - self._mfb = buffer(self._mf, ofs, size) - #END handle buffer wrapping - finally: - os.close(fd) - #END close file handle - - def ofs_begin(self): - """:return: absolute byte offset to the first byte of the mapping""" - return self._b - - def size(self): - """:return: total size of the mapped region in bytes""" - return len(self._mf) + def __init__(self, manager = None, regions = None): + self._manager = manager + self._regions = regions + self._region = region + self._ofs = 0 + self._size = 0 - def ofs_end(self): - """:return: Absolute offset to one byte beyond the mapping into the file""" - return self._b + self.size() + def __del__(self): + self._destroy() - def includes_ofs(self, ofs): - """:return: True if the given offset can be read in our mapped region""" - return (ofs >= self.ofs_begin()) and (ofs <= self.ofs_end()) + def _destroy(self): + """Destruction code to decrement counters""" - def client_count(self): - """:return: number of clients currently using this region""" - # -1: self on stack, -1 self in this method, -1 self in getrefcount - return getrefcount(self)-3 + def _copy_from(self, rhs): + """Copy all data from rhs into this instance, handles usage count""" - def adjust_client_count(self, ofs): - """Adjust the client count by the given positive or negative offset""" - self._nc += ofs - - def usage_count(self): - """:return: amount of usages so far""" - return self._uc - - def adjust_usage_count(self, ofs): - """Adjust the usage count by the given positive or negative offset""" - self._uc += ofs - - # re-define all methods which need offset adjustments in compatibility mode - if _need_compat_layer: - def size(self): - return len(self._mf) - self._b - - def ofs_end(self): - return len(self._mf) - #END handle compat layer + #{ Interface - -class MappedRegionList(list): - """List of MappedRegion instances associating a path with a list of regions.""" - __slots__ = ( - '_path', # path which is mapped by all our regions - '_file_size' # total size of the file we map - ) - - def __new__(cls, path): - return super(MappedRegionList, cls).__new__(cls) - def __init__(self, path): - self._path = path - self._file_size = None - - def path(self): - """:return: path to file whose regions we manage""" - return self._path - - def file_size(self): - """:return: size of file we manager""" - if self._file_size is None: - self._file_size = os.stat(self._path).st_size - #END update file size - return self._file_size + #} END interface - -class Cursor(object): - """Pointer into the mapped region of the memory manager, keeping the current window - alive until it is destroyed""" class MappedMemoryManager(object): diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 0605313d3..6957dcab0 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -49,7 +49,7 @@ class TestBase(TestCase): """Foundation used by all tests""" #{ Configuration - + k_window_test_size = 1000 * 1000 * 8 + 5195 #} END configuration #{ Overrides diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 2027d9004..edc2ec2fe 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,102 +1,11 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import align_to_page -from smmap.mman import Window -from smmap.mman import MappedRegion -from smmap.mman import MappedRegionList -from smmap.mman import Cursor - -import sys -import mmap class TestMMan(TestBase): - _window_test_size = 1000 * 1000 * 8 + 5195 - - def test_window(self): - wl = Window(0, 1) # left - wc = Window(1, 1) # center - wc2 = Window(10, 5) # another center - wr = Window(8000, 50) # right - - assert wl.ofs_end() == 1 - assert wc.ofs_end() == 2 - assert wr.ofs_end() == 8050 - - # extension does nothing if already in place - maxsize = 100 - wc.extend_left_to(wl, maxsize) - assert wc.ofs == 1 and wc.size == 1 - wl.extend_right_to(wc, maxsize) - wl.extend_right_to(wc, maxsize) - assert wl.ofs == 0 and wl.size == 1 - - # an actual left extension - pofs_end = wc2.ofs_end() - wc2.extend_left_to(wc, maxsize) - assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() - - - # respects maxsize - wc.extend_right_to(wr, maxsize) - assert wc.ofs == 1 and wc.size == maxsize - wc.extend_right_to(wr, maxsize) - assert wc.ofs == 1 and wc.size == maxsize - - # without maxsize - wc.extend_right_to(wr, sys.maxint) - assert wc.ofs_end() == wr.ofs and wc.ofs == 1 - - # extend left - wr.extend_left_to(wc2, maxsize) - wr.extend_left_to(wc2, maxsize) - assert wr.size == maxsize - - wr.extend_left_to(wc2, sys.maxint) - assert wr.ofs == wc2.ofs_end() - - wc.align() - assert wc.ofs == 0 and wc.size == mmap.PAGESIZE*2 - - def test_region(self): - fc = FileCreator(self._window_test_size, "window_test") - half_size = fc.size / 2 - rofs = align_to_page(4200, False) - rfull = MappedRegion(fc.path, 0, fc.size) - rhalfofs = MappedRegion(fc.path, rofs, fc.size) - rhalfsize = MappedRegion(fc.path, 0, half_size) - - # offsets - assert rfull.ofs_begin() == 0 and rfull.size() == fc.size - assert rfull.ofs_end() == fc.size # if this method works, it works always - - assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs - assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size - - assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) - assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) - assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) - - # auto-refcount - assert rfull.client_count() == 1 - rfull2 = rfull - assert rfull.client_count() == 2 - - # window constructor - w = Window.from_region(rfull) - assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() - - def test_region_list(self): - fc = FileCreator(100, "sample_file") - ml = MappedRegionList(fc.path) - - assert len(ml) == 0 - assert ml.path() == fc.path - assert ml.file_size() == fc.size - def test_cursor(self): - pass + man = MappedMemoryManager() def test_basics(self): pass diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py new file mode 100644 index 000000000..815391613 --- /dev/null +++ b/smmap/test/test_util.py @@ -0,0 +1,89 @@ +from lib import TestBase, FileCreator + +from smmap.util import * + +import sys + +class TestMMan(TestBase): + + def test_window(self): + wl = MemoryWindow(0, 1) # left + wc = MemoryWindow(1, 1) # center + wc2 = MemoryWindow(10, 5) # another center + wr = MemoryWindow(8000, 50) # right + + assert wl.ofs_end() == 1 + assert wc.ofs_end() == 2 + assert wr.ofs_end() == 8050 + + # extension does nothing if already in place + maxsize = 100 + wc.extend_left_to(wl, maxsize) + assert wc.ofs == 1 and wc.size == 1 + wl.extend_right_to(wc, maxsize) + wl.extend_right_to(wc, maxsize) + assert wl.ofs == 0 and wl.size == 1 + + # an actual left extension + pofs_end = wc2.ofs_end() + wc2.extend_left_to(wc, maxsize) + assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() + + + # respects maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + + # without maxsize + wc.extend_right_to(wr, sys.maxint) + assert wc.ofs_end() == wr.ofs and wc.ofs == 1 + + # extend left + wr.extend_left_to(wc2, maxsize) + wr.extend_left_to(wc2, maxsize) + assert wr.size == maxsize + + wr.extend_left_to(wc2, sys.maxint) + assert wr.ofs == wc2.ofs_end() + + wc.align() + assert wc.ofs == 0 and wc.size == PAGESIZE*2 + + def test_region(self): + fc = FileCreator(self.k_window_test_size, "window_test") + half_size = fc.size / 2 + rofs = align_to_page(4200, False) + rfull = MappedRegion(fc.path, 0, fc.size) + rhalfofs = MappedRegion(fc.path, rofs, fc.size) + rhalfsize = MappedRegion(fc.path, 0, half_size) + + # offsets + assert rfull.ofs_begin() == 0 and rfull.size() == fc.size + assert rfull.ofs_end() == fc.size # if this method works, it works always + + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs + assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size + + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) + assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + + # auto-refcount + assert rfull.client_count() == 1 + rfull2 = rfull + assert rfull.client_count() == 2 + + # window constructor + w = MemoryWindow.from_region(rfull) + assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() + + def test_region_list(self): + fc = FileCreator(100, "sample_file") + ml = MappedRegionList(fc.path) + + assert len(ml) == 0 + assert ml.path() == fc.path + assert ml.file_size() == fc.size + diff --git a/smmap/util.py b/smmap/util.py new file mode 100644 index 000000000..784227752 --- /dev/null +++ b/smmap/util.py @@ -0,0 +1,189 @@ +"""Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" +import os +import sys +import mmap + +from mmap import PAGESIZE +from sys import getrefcount + +__all__ = ["align_to_page", "MemoryWindow", "MappedRegion", "MappedRegionList", "PAGESIZE"] + +#{ Utilities + +def align_to_page(num, round_up): + """Align the given integer number to the closest page offset, which usually is 4096 bytes. + :param round_up: if True, the next higher multiple of page size is used, otherwise + the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) + :return: num rounded to closest page""" + res = (num / PAGESIZE) * PAGESIZE; + if round_up and (res != num): + res += PAGESIZE; + #END handle size + return res; + +#}END utilities + + +#{ Utility Classes + +class MemoryWindow(object): + """Utility type which is used to snap windows towards each other, and to adjust their size""" + __slots__ = ( + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) + + def __init__(self, offset, size): + self.ofs = offset + self.size = size + + def __repr__(self): + return "MemoryWindow(%i, %i)" % (self.ofs, self.size) + + @classmethod + def from_region(cls, region): + """:return: new window from a region""" + return cls(region.ofs_begin(), region.size()) + + def ofs_end(self): + return self.ofs + self.size + + def align(self): + self.ofs = align_to_page(self.ofs, 0) + self.size = align_to_page(self.size, 1) + + def extend_left_to(self, window, max_size): + """Adjust the offset to start where the given window on our left ends if possible, + but don't make yourself larger than max_size. + The resize will assure that the new window still contains the old window area""" + rofs = self.ofs - window.ofs_end() + nsize = rofs + self.size + rofs -= nsize - min(nsize, max_size) + self.ofs = self.ofs - rofs + self.size += rofs + + def extend_right_to(self, window, max_size): + """Adjust the size to make our window end where the right window begins, but don't + get larger than max_size""" + self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) + + +class MappedRegion(object): + """Defines a mapped region of memory, aligned to pagesizes + :note: deallocates used region automatically on destruction""" + __slots__ = [ + '_b' , # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_uc', # total amount of usages + '_ms', # actual size of the mapping + '__weakref__' # allow weak references to a region + ] + _need_compat_layer = sys.version_info[1] < 6 + + if _need_compat_layer: + __slots__.append('_mfb') # mapped memory buffer to provide offset + #END handle additional slot + + + def __init__(self, path, ofs, size): + """Initialize a region, allocate the memory map + :param path: path to the file to map + :param ofs: **aligned** offset into the file to be mapped + :param size: if size is larger then the file on disk, the whole file will be + allocated the the size automatically adjusted + :raise Exception: if no memory can be allocated""" + self._b = ofs + self._uc = 0 + + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + try: + kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) + corrected_size = size + sizeofs = ofs + if self._need_compat_layer: + del(kwargs['offset']) + corrected_size += ofs + sizeofs = 0 + # END handle python not supporting offset ! Arg + + # have to correct size, otherwise (instead of the c version) it will + # bark that the size is too large ... many extra file accesses because + # if this ... argh ! + self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size - sizeofs), **kwargs) + + if self._need_compat_layer: + self._mfb = buffer(self._mf, ofs, size) + #END handle buffer wrapping + finally: + os.close(fd) + #END close file handle + + def ofs_begin(self): + """:return: absolute byte offset to the first byte of the mapping""" + return self._b + + def size(self): + """:return: total size of the mapped region in bytes""" + return len(self._mf) + + def ofs_end(self): + """:return: Absolute offset to one byte beyond the mapping into the file""" + return self._b + self.size() + + def includes_ofs(self, ofs): + """:return: True if the given offset can be read in our mapped region""" + return (ofs >= self.ofs_begin()) and (ofs <= self.ofs_end()) + + def client_count(self): + """:return: number of clients currently using this region""" + # -1: self on stack, -1 self in this method, -1 self in getrefcount + return getrefcount(self)-3 + + def adjust_client_count(self, ofs): + """Adjust the client count by the given positive or negative offset""" + self._nc += ofs + + def usage_count(self): + """:return: amount of usages so far""" + return self._uc + + def adjust_usage_count(self, ofs): + """Adjust the usage count by the given positive or negative offset""" + self._uc += ofs + + # re-define all methods which need offset adjustments in compatibility mode + if _need_compat_layer: + def size(self): + return len(self._mf) - self._b + + def ofs_end(self): + return len(self._mf) + #END handle compat layer + + +class MappedRegionList(list): + """List of MappedRegion instances associating a path with a list of regions.""" + __slots__ = ( + '_path', # path which is mapped by all our regions + '_file_size' # total size of the file we map + ) + + def __new__(cls, path): + return super(MappedRegionList, cls).__new__(cls) + + def __init__(self, path): + self._path = path + self._file_size = None + + def path(self): + """:return: path to file whose regions we manage""" + return self._path + + def file_size(self): + """:return: size of file we manager""" + if self._file_size is None: + self._file_size = os.stat(self._path).st_size + #END update file size + return self._file_size + +#} END utilty classes From 9fe8f93439118c7ea6c4d5dece879b3849f0931e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 8 Jun 2011 23:50:19 +0200 Subject: [PATCH 0147/1392] Implemented first basic functionality of cursor, which is only complete once some more of the memory manager is implemented. Next up is its major use_window method --- smmap/mman.py | 106 +++++++++++++++++++++++++++++++++++++--- smmap/test/test_mman.py | 15 +++++- smmap/test/test_util.py | 5 ++ smmap/util.py | 12 ++--- 4 files changed, 120 insertions(+), 18 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 0ecc1aad1..af414eada 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,29 +1,34 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" - -__all__ = ["MappedMemoryManager", "MemoryCursor"] - from util import ( MemoryWindow, MappedRegion, MappedRegionList, ) +from weakref import proxy + +__all__ = ["MappedMemoryManager"] +#{ Utilities + +#}END utilities class MemoryCursor(object): """Pointer into the mapped region of the memory manager, keeping the current window - alive until it is destroyed""" + alive until it is destroyed. + + Cursors should not be created manually, but are instead returned by the MappedMemoryManager""" __slots__ = ( '_manager', # the manger keeping all file regions - '_regions', # a regions list with regions for our file - '_region', # WEAK REF to our current region + '_rlist', # a regions list with regions for our file + '_region', # our current region or None '_ofs', # relative offset from the actually mapped area to our start area '_size' # maximum size we should provide ) def __init__(self, manager = None, regions = None): self._manager = manager - self._regions = regions - self._region = region + self._rlist = regions + self._region = None self._ofs = 0 self._size = 0 @@ -32,12 +37,97 @@ def __del__(self): def _destroy(self): """Destruction code to decrement counters""" + self.unuse_region() + + if self._rlist is not None: + # Actual client count, which doesn't include the reference kept by the manager, nor ours + # as we are about to be deleted + num_clients = self._rlist.client_count() - 2 + if num_clients == 0 and len(self._rlist) == 0: + # Free all resources associated with the mapped file + self._manager._files.pop(self._rlist.path()) + #END remove regions list from manager + #END handle regions def _copy_from(self, rhs): """Copy all data from rhs into this instance, handles usage count""" + self._manager = rhs._manager + self._rlist = rhs._rlist + self._region = rhs._region + self._ofs = rhs._ofs + self._size = rhs._size + + if self._region is not None: + self._region.increment_usage_count(1) + # END handle regions + + def __copy__(self): + """copy module interface""" + cpy = type(self)() + cpy._copy_from(self) + return cpy #{ Interface + def assign(self, rhs): + """Assign rhs to this instance. This is required in order to get a real copy. + Alternativly, you can copy an existing instance using the copy module""" + self._destroy() + self._copy_from(rhs) + + def use_region(self, offset, size): + """Assure we point to a window which allows access to the given offset into the file + :param offset: absolute offset in bytes into the file + :param size: amount of bytes to map + :return: this instance - it should be queried for whether it points to a valid memory region. + This is not the case if the mapping failed becaues we reached the end of the file + :note: The size actually mapped may be smaller than the given size. If that is the case, + either the file has reached its end, or the map was created between two existing regions""" + + def unuse_region(self): + """Unuse the ucrrent region. Does nothing if we have no current region + :note: the cursor unuses the region automatically upon destruction. It is recommended + to unuse the region once you are done reading from it in persistent cursors as it + helps to free up resource more quickly""" + self._region = None + def is_valid(self): + """:return: True if we have a valid and usable region""" + return self._region is not None + + def is_associated(self): + """:return: True if we are associated with a specific file already""" + return self._rlist is not None + + def ofs_begin(self): + """:return: offset to the first byte pointed to by our cursor""" + return self._region.ofs_begin() + self._ofs + + def size(self): + """:return: amount of bytes we point to""" + return self._size + + def region_ref(self): + """:return: weak proxy to our mapped region. + :raise AssertionError: if we have no current region. This is only useful for debugging""" + if self._region is None: + raise AssertionError("region not set") + return proxy(self._region) + + def includes_ofs(self, ofs): + """:return: True if the given absolute offset is contained in the cursors + current region + :note: always False if the cursor does not point to a valid region""" + if self._region is None: + return False + return (self.ofs_begin() <= ofs) and (ofs < self.ofs_end()) + + def file_size(self): + """:return: size of the underlying file""" + return self._rlist.file_size() + + def path(self): + """:return: path of the underlying mapped file""" + return self._rlist.path() #} END interface diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index edc2ec2fe..85a0a1bf5 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,11 +1,22 @@ from lib import TestBase, FileCreator +from copy import copy from smmap.mman import * +from smmap.mman import MemoryCursor class TestMMan(TestBase): def test_cursor(self): - man = MappedMemoryManager() + man = MappedMemoryManager() + c = MemoryCursor(man) + assert not c.is_valid() + assert not c.is_associated() - def test_basics(self): + # copy module + + # assign method + + + + def test_memory_manager(self): pass diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 815391613..2a0e0551e 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -75,6 +75,11 @@ def test_region(self): rfull2 = rfull assert rfull.client_count() == 2 + # usage + assert rfull.usage_count() == 0 + rfull.increment_usage_count() + assert rfull.usage_count() == 1 + # window constructor w = MemoryWindow.from_region(rfull) assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() diff --git a/smmap/util.py b/smmap/util.py index 784227752..e4c5c82d6 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -75,8 +75,8 @@ class MappedRegion(object): '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_uc', # total amount of usages - '_ms', # actual size of the mapping - '__weakref__' # allow weak references to a region + '_ms' # actual size of the mapping + '__weakref__' ] _need_compat_layer = sys.version_info[1] < 6 @@ -139,17 +139,13 @@ def client_count(self): # -1: self on stack, -1 self in this method, -1 self in getrefcount return getrefcount(self)-3 - def adjust_client_count(self, ofs): - """Adjust the client count by the given positive or negative offset""" - self._nc += ofs - def usage_count(self): """:return: amount of usages so far""" return self._uc - def adjust_usage_count(self, ofs): + def increment_usage_count(self): """Adjust the usage count by the given positive or negative offset""" - self._uc += ofs + self._uc += 1 # re-define all methods which need offset adjustments in compatibility mode if _need_compat_layer: From 226f42858faab545df0e0a9636c84791f0b3a080 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 01:13:52 +0200 Subject: [PATCH 0148/1392] First bit of MappedMemoryManager started. Much more to come - got lost in git++ and improved it in the meanwhile --- smmap/exc.py | 7 +++++++ smmap/mman.py | 22 +++++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 smmap/exc.py diff --git a/smmap/exc.py b/smmap/exc.py new file mode 100644 index 000000000..a090d24d5 --- /dev/null +++ b/smmap/exc.py @@ -0,0 +1,7 @@ +"""Module with system exceptions""" + +class MemoryManagerError(Exception): + """Base class for all exceptions thrown by the memory manager""" + +class RegionCollectionError(MemoryManagerError): + """Thrown if a memory region could not be collected, or if no region for collection was found""" diff --git a/smmap/mman.py b/smmap/mman.py index af414eada..0d8c8dce4 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -5,6 +5,7 @@ MappedRegionList, ) +from exc import RegionCollectionError from weakref import proxy __all__ = ["MappedMemoryManager"] @@ -45,7 +46,7 @@ def _destroy(self): num_clients = self._rlist.client_count() - 2 if num_clients == 0 and len(self._rlist) == 0: # Free all resources associated with the mapped file - self._manager._files.pop(self._rlist.path()) + self._manager._fdict.pop(self._rlist.path()) #END remove regions list from manager #END handle regions @@ -82,6 +83,7 @@ def use_region(self, offset, size): This is not the case if the mapping failed becaues we reached the end of the file :note: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" + def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region @@ -146,5 +148,19 @@ class MappedMemoryManager(object): a safe amount of memory already, which would possibly cause memory allocations to fail as our address space is full.""" - __slots__ = tuple() - + __slots__ = [ + '_fdict', # mapping of path -> MappedRegionList + '_max_window_size', # maximum size of a window + '_max_memory_size', # maximum amount ofmemory we may allocate + '_max_handles', # maximum amount of handles to keep open + '_memory_size', # currently allocated memory size + '_handle_count', # amount of currently allocated file handles + ] + + def _collect_one_lru_region(self, size): + """Unmap the region which was least-recently used and has no client + :param size: size of the region we want to map next (assuming its not already mapped partially or full + if 0, we try to free any available region + :raise RegionCollectionError: + :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + From 04b9ec1e2c0bf657bf575a72a27acdbf2935ecf4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 11:14:06 +0200 Subject: [PATCH 0149/1392] Fully implemented the manager - now the cursor can be implenented as well --- smmap/mman.py | 102 +++++++++++++++++++++++++++++++++++++++- smmap/test/test_mman.py | 22 ++++++++- smmap/test/test_util.py | 5 ++ smmap/util.py | 7 ++- 4 files changed, 131 insertions(+), 5 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 0d8c8dce4..9c5432bd1 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -3,6 +3,8 @@ MemoryWindow, MappedRegion, MappedRegionList, + is_64_bit, + PAGESIZE ) from exc import RegionCollectionError @@ -150,17 +152,113 @@ class MappedMemoryManager(object): __slots__ = [ '_fdict', # mapping of path -> MappedRegionList - '_max_window_size', # maximum size of a window + '_window_size', # maximum size of a window '_max_memory_size', # maximum amount ofmemory we may allocate - '_max_handles', # maximum amount of handles to keep open + '_max_handle_count', # maximum amount of handles to keep open '_memory_size', # currently allocated memory size '_handle_count', # amount of currently allocated file handles ] + _MB_in_bytes = 1024 * 1024 + + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = ~0): + """initialize the manager with the given parameters. + :param window_size: if 0, a default window size will be chosen depending on + the operating system's architechture. It will internally be quantified to a multiple of the page size + :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. + If 0, a viable default iwll be set dependning on the system's architecture. + :param max_open_handles: if not ~0, lmit the amount of open file handles to the given number. + Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, + the manager will free as many handles as posisble""" + self._fdict = dict() + self._window_size = window_size + self._max_memory_size = max_memory_size + self._max_handle_count = max_open_handles + self._memory_size = 0 + self._handle_count = 0 + + if window_size == 0: + coeff = 32 + if is_64_bit(): + coeff = 1024 + #END handle arch + self._window_size = coeff * self._MB_in_bytes + # END handle max window size + + if max_memory_size == 0: + coeff = 512 + if is_64_bit(): + coeff = 8192 + #END handle arch + self._max_memory_size = coeff * self._MB_in_bytes + #END handle max memory size + def _collect_one_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region :raise RegionCollectionError: :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + num_found = 0 + while (size == 0) or (self._memory_size + size > self._max_memory_size): + lru_region = None + lru_list = None + for regions in self._fdict.itervalues(): + for region in regions: + # check client count - consider that we keep one reference ourselves ! + if (region.client_count()-1 == 0 and + (lru_region is None or region.usage_count() < lru_region.usage_count())): + lru_region = region + lru_list = regions + # END update lru_region + #END for each region + #END for each regions list + + if lru_region is None: + if num_found == 0 and size != 0: + raise RegionCollectionError("Didn't find any region to free") + #END raise if necessary + break + #END handle region not found + + num_found += 1 + del(lru_list[lru_list.index(lru_region)]) + self._memory_size -= lru_region.size() + self._handle_count -= 1 + #END while there is more memory to free + + #{ Interface + def make_cursor(self, path): + """:return: a cursor pointing to the given path. It can be used to map new regions of the file into memory""" + regions = self._fdict.get(path) + if regions is None: + regions = MappedRegionList(path) + self._fdict[path] = regions + # END obtain region for path + return MemoryCursor(self, regions) + def num_file_handles(self): + """:return: amount of file handles in use. Each mapped region uses one file handle""" + return self._handle_count + + def num_open_files(self): + """Amount of opened files in the system""" + return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) + + def window_size(self): + """:return: size of each window when allocating new regions""" + return self._window_size + + def mapped_memory_size(self): + """:return: amount of bytes currently mapped in total""" + return self._memory_size + + def max_mapped_memory_size(self): + """:return: maximum amount of memory we may allocate""" + return self._max_memory_size + + def page_size(self): + """:return: size of a single memory page in bytes""" + return PAGESIZE + + #} END interface diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 85a0a1bf5..92c201c78 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,8 +1,13 @@ from lib import TestBase, FileCreator -from copy import copy from smmap.mman import * from smmap.mman import MemoryCursor +from smmap.util import PAGESIZE + +from smmap.exc import RegionCollectionError + +import sys +from copy import copy class TestMMan(TestBase): @@ -19,4 +24,17 @@ def test_cursor(self): def test_memory_manager(self): - pass + man = MappedMemoryManager() + assert man.num_file_handles() == 0 + assert man.num_open_files() == 0 + assert man.window_size() > 0 + assert man.mapped_memory_size() == 0 + assert man.max_mapped_memory_size() > 0 + assert man.page_size() == PAGESIZE + + # collection doesn't raise in 'any' mode + man._collect_one_lru_region(0) + # doesn't raise if we are within the limit + man._collect_one_lru_region(10) + # raises outside of limit + self.failUnlessRaises(RegionCollectionError, man._collect_one_lru_region, sys.maxint) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 2a0e0551e..9866217d1 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -92,3 +92,8 @@ def test_region_list(self): assert ml.path() == fc.path assert ml.file_size() == fc.size + def test_util(self): + assert isinstance(is_64_bit(), bool) # just call it + assert align_to_page(1, False) == 0 + assert align_to_page(1, True) == PAGESIZE + diff --git a/smmap/util.py b/smmap/util.py index e4c5c82d6..7f42834c4 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -6,7 +6,8 @@ from mmap import PAGESIZE from sys import getrefcount -__all__ = ["align_to_page", "MemoryWindow", "MappedRegion", "MappedRegionList", "PAGESIZE"] +__all__ = [ "align_to_page", "is_64_bit", + "MemoryWindow", "MappedRegion", "MappedRegionList", "PAGESIZE"] #{ Utilities @@ -20,6 +21,10 @@ def align_to_page(num, round_up): res += PAGESIZE; #END handle size return res; + +def is_64_bit(): + """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" + return sys.maxint > (1<<32) - 1 #}END utilities From e7b0e1c2c55c52b144671a1f3a8433901e42804a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 12:56:56 +0200 Subject: [PATCH 0150/1392] Implemented use_region. Let the testing begin. Especially the actual data handling will be interesting, which has to work exclusively through buffer objects. There are plenty of layers between the user and the data, which will always be copied when slicing it (as we have no memoryview). The latter one could be implemented on in case we use 2.7 at some point. Now, let the testing begin ! --- smmap/mman.py | 131 ++++++++++++++++++++++++++++++++++++++-- smmap/test/test_mman.py | 38 ++++++++++-- smmap/test/test_util.py | 2 + smmap/util.py | 16 ++++- 4 files changed, 174 insertions(+), 13 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 9c5432bd1..1261fc16c 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -9,6 +9,7 @@ from exc import RegionCollectionError from weakref import proxy +import sys __all__ = ["MappedMemoryManager"] #{ Utilities @@ -77,7 +78,7 @@ def assign(self, rhs): self._destroy() self._copy_from(rhs) - def use_region(self, offset, size): + def use_region(self, offset, size, _is_recursive=False): """Assure we point to a window which allows access to the given offset into the file :param offset: absolute offset in bytes into the file :param size: amount of bytes to map @@ -85,15 +86,130 @@ def use_region(self, offset, size): This is not the case if the mapping failed becaues we reached the end of the file :note: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" + need_region = True + man = self._manager + size = min(size, man.window_size()) # clamp size to window size + if self._region is not None: + if self._region.includes_ofs(offset): + need_region = False + else: + self.unuse_region() + # END handle existing region + # END check existing region + + if need_region: + # abort on offsets beyond our mapped file's size - currently we are invalid + if offset > self.file_size(): + return self + # END handle offset too large + + existing_region = None + for region in self._rlist: + if region.includes_ofs(offset): + existing_region = region + break + #END handle existing region + #END for each existing region + if existing_region is None: + left = MemoryWindow(0, 0) + mid = MemoryWindow(offset, size) + right = MemoryWindow(self.file_size(), 0) + + # we want to honor the max memory size, and assure we have anough + # memory available + man._collect_lru_region(man.window_size()) + + # we assume the list remains sorted by offset + insert_pos = 0 + len_regions = len(self._rlist) + if len_regions == 1: + if self._rlist[0].ofs_begin() <= offset: + insert_pos = 1 + #END maintain sort + else: + # find insert position + insert_pos = len_regions + for i, region in enumerate(self._rlist): + if region.ofs_begin() > offset: + insert_pos = i + break + #END if insert position is correct + #END for each region + # END obtain insert pos + + # adjust the actual offset and size values to create the largest + # possible mapping + if insert_pos == 0: + if len_regions: + right = MemoryWindow.from_region(self._rlist[insert_pos]) + #END adjust right side + else: + if insert_pos != len_regions: + right = MemoryWindow.from_region(self._rlist[insert_pos]) + # END adjust right window + left = MemoryWindow.from_region(self._rlist[insert_pos - 1]) + #END adjust surrounding windows + + mid.extend_left_to(left, man._window_size) + mid.extend_right_to(right, man._window_size) + mid.align() + + # it can happen that we align beyond the end of the file + if mid.ofs_end() > right.ofs: + mid.size = right.ofs - mid.ofs + #END readjust size + + # insert new region at the right offset to keep the order + try: + if man._handle_count >= man._max_handle_count: + raise Exception + #END assert own imposed max file handles + self._region = MappedRegion(self._rlist.path(), mid.ofs, mid.size) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if _is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + man._collect_lru_region(0) + return self.use_region(offset, size, True) + #END handle exceptions + + man._handle_count += 1 + man._memory_size += self._region.size() + self._rlist.insert(insert_pos, self._region) + else: + self._region = existing_region + #END need region handling + #END handle acquire region + + self._region.increment_usage_count() + self._ofs = offset - self._region.ofs_begin() + self._size = min(size, self._region.ofs_end() - offset) + + return self + def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region :note: the cursor unuses the region automatically upon destruction. It is recommended to unuse the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None - + + def buffer(self): + """Return a buffer object which allows access to our memory region from our offset + to the window size. Please note that it might be smaller than you requested + :note: You can only obtain a buffer if this instance is_valid() !""" + return buffer(self._region.buffer(), self._ofs, self._size) + def is_valid(self): """:return: True if we have a valid and usable region""" return self._region is not None @@ -136,7 +252,6 @@ def path(self): #} END interface - class MappedMemoryManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. @@ -161,13 +276,13 @@ class MappedMemoryManager(object): _MB_in_bytes = 1024 * 1024 - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = ~0): + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): """initialize the manager with the given parameters. :param window_size: if 0, a default window size will be chosen depending on the operating system's architechture. It will internally be quantified to a multiple of the page size :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default iwll be set dependning on the system's architecture. - :param max_open_handles: if not ~0, lmit the amount of open file handles to the given number. + :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, the manager will free as many handles as posisble""" self._fdict = dict() @@ -193,7 +308,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = ~0): self._max_memory_size = coeff * self._MB_in_bytes #END handle max memory size - def _collect_one_lru_region(self, size): + def _collect_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region @@ -253,6 +368,10 @@ def mapped_memory_size(self): """:return: amount of bytes currently mapped in total""" return self._memory_size + def max_file_handles(self): + """:return: maximium amount of handles we may have opened""" + return self._max_handle_count + def max_mapped_memory_size(self): """:return: maximum amount of memory we may allocate""" return self._max_memory_size diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 92c201c78..97cd8f62b 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -12,16 +12,36 @@ class TestMMan(TestBase): def test_cursor(self): + fc = FileCreator(self.k_window_test_size, "cursor_test") + man = MappedMemoryManager() - c = MemoryCursor(man) - assert not c.is_valid() - assert not c.is_associated() + ci = MemoryCursor(man) # invalid cursor + assert not ci.is_valid() + assert not ci.is_associated() + assert ci.size() == 0 # this is cached, so we can query it in invalid state + + cv = man.make_cursor(fc.path) + assert not cv.is_valid() # no region mapped yet + assert cv.is_associated()# but it know where to map it from + assert cv.file_size() == fc.size + assert cv.path() == fc.path # copy module + cio = copy(cv) + assert not cio.is_valid() and cio.is_associated() # assign method + assert not ci.is_associated() + ci.assign(cv) + assert not ci.is_valid() and ci.is_associated() + # unuse non-existing region is fine + cv.unuse_region() + cv.unuse_region() + # destruction is fine (even multiple times) + cv._destroy() + MemoryCursor(man)._destroy() def test_memory_manager(self): man = MappedMemoryManager() @@ -33,8 +53,14 @@ def test_memory_manager(self): assert man.page_size() == PAGESIZE # collection doesn't raise in 'any' mode - man._collect_one_lru_region(0) + man._collect_lru_region(0) # doesn't raise if we are within the limit - man._collect_one_lru_region(10) + man._collect_lru_region(10) # raises outside of limit - self.failUnlessRaises(RegionCollectionError, man._collect_one_lru_region, sys.maxint) + self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) + + + # use a region, verify most basic functionality + fc = FileCreator(self.k_window_test_size, "manager_test") + c = man.make_cursor(fc.path) + assert c.use_region(10, 10).is_valid() diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 9866217d1..136d99102 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -88,6 +88,8 @@ def test_region_list(self): fc = FileCreator(100, "sample_file") ml = MappedRegionList(fc.path) + assert ml.client_count() == 1 + assert len(ml) == 0 assert ml.path() == fc.path assert ml.file_size() == fc.size diff --git a/smmap/util.py b/smmap/util.py index 7f42834c4..456562a5f 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -54,7 +54,10 @@ def ofs_end(self): return self.ofs + self.size def align(self): - self.ofs = align_to_page(self.ofs, 0) + """Assures the previous window area is contained in the new one""" + nofs = align_to_page(self.ofs, 0) + self.size += self.ofs - nofs # keep size constant + self.ofs = nofs self.size = align_to_page(self.size, 1) def extend_left_to(self, window, max_size): @@ -123,6 +126,10 @@ def __init__(self, path, ofs, size): os.close(fd) #END close file handle + def buffer(self): + """:return: a sliceable buffer which can be used to access the mapped memory""" + return self._mf + def ofs_begin(self): """:return: absolute byte offset to the first byte of the mapping""" return self._b @@ -159,6 +166,9 @@ def size(self): def ofs_end(self): return len(self._mf) + + def buffer(self): + return self._mfb #END handle compat layer @@ -176,6 +186,10 @@ def __init__(self, path): self._path = path self._file_size = None + def client_count(self): + """:return: amount of clients which hold a reference to this instance""" + return getrefcount(self)-3 + def path(self): """:return: path to file whose regions we manage""" return self._path From 997a580c0a14cfa0bed11477cd64198199ef99b6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 15:29:52 +0200 Subject: [PATCH 0151/1392] implemented plenty of operational testing. It shows that its not yet working properly. Intersting, it was so promising, and went just a little bit too smooth. There we go :) --- smmap/mman.py | 13 ++++-- smmap/test/test_mman.py | 100 ++++++++++++++++++++++++++++++++++++++-- smmap/util.py | 1 - 3 files changed, 106 insertions(+), 8 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 1261fc16c..8a9066cdc 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -8,7 +8,7 @@ ) from exc import RegionCollectionError -from weakref import proxy +from weakref import ref import sys __all__ = ["MappedMemoryManager"] @@ -100,7 +100,7 @@ def use_region(self, offset, size, _is_recursive=False): if need_region: # abort on offsets beyond our mapped file's size - currently we are invalid - if offset > self.file_size(): + if offset >= self.file_size(): return self # END handle offset too large @@ -222,16 +222,21 @@ def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor""" return self._region.ofs_begin() + self._ofs + def ofs_end(self): + """:return: offset to one past the last available byte""" + # unroll method calls for performance ! + return self._region.ofs_begin() + self._ofs + self._size + def size(self): """:return: amount of bytes we point to""" return self._size def region_ref(self): - """:return: weak proxy to our mapped region. + """:return: weak ref to our mapped region. :raise AssertionError: if we have no current region. This is only useful for debugging""" if self._region is None: raise AssertionError("region not set") - return proxy(self._region) + return ref(self._region) def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 97cd8f62b..f592fc129 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -2,10 +2,11 @@ from smmap.mman import * from smmap.mman import MemoryCursor -from smmap.util import PAGESIZE - +from smmap.util import PAGESIZE, align_to_page from smmap.exc import RegionCollectionError +from random import randint +from time import time import sys from copy import copy @@ -59,8 +60,101 @@ def test_memory_manager(self): # raises outside of limit self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) - # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") c = man.make_cursor(fc.path) assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + + def test_memman_operation(self): + # test more access, force it to actually unmap regions + fc = FileCreator(self.k_window_test_size, "manager_operation_test") + data = open(fc.path, 'rb').read() + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + man = MappedMemoryManager(fc.size / 100, fc.size / 3, 15) + c = man.make_cursor(fc.path) + + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + size = man.window_size() / 2 + assert c.use_region(base_offset, size).is_valid() + rr = c.region_ref() + assert rr().client_count() == 2 # the manager and the cursor and us + + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr().size() + assert c.size() == size + assert c.ofs_begin() == base_offset + assert rr().ofs_begin() == 0 # it was aligned and expanded + assert rr().size() == align_to_page(man.window_size(), True) # but isn't larger than the max window (aligned) + + assert c.buffer()[:] == data[base_offset:base_offset+size] + + # obtain second window, which spans the first part of the file - it is a still the same window + assert c.use_region(0, size-10).is_valid() + assert c.region_ref()() == rr() + assert man.num_file_handles() == 1 + assert c.size() == size-10 + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:size-10] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - size + overshoot + assert c.use_region(base_offset, size).is_valid() + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager + rr = c.region_ref() + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset+size] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 15000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + while num_random_accesses: + num_random_accesses += 1 + base_offset = randint(0, fc.size - 1) + + # precondition + assert man.max_mapped_memory_size() >= man.mapped_memory_size() + assert man.max_file_handles() >= man.num_file_handles() + + assert c.use_region(base_offset, size).is_valid() + assert c.buffer()[:] == data[base_offset:base_offset+size] + memory_read += c.size() + + assert c.includes_ofs(base_offset) + assert c.includes_ofs(base_offset+c.size()-1) + assert not c.includes_ofs(base_offset+c.size()) + # END while we should do an access + elapsed = time() - st + mb = 1000 * 1000 + sys.stderr.write("Read %i mb of memory with %i random accesses in %f s(%f mb/s)\n" + % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) + diff --git a/smmap/util.py b/smmap/util.py index 456562a5f..55e007554 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -83,7 +83,6 @@ class MappedRegion(object): '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_uc', # total amount of usages - '_ms' # actual size of the mapping '__weakref__' ] _need_compat_layer = sys.version_info[1] < 6 From c90cfef5597b403ae03edf021442f9f44139561f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 15:56:43 +0200 Subject: [PATCH 0152/1392] Fixed a few little issues, the random access test/perftest now work perfectly. Its not too slow either, 160MB/s compared to the 320MB/s that the c++ implementation gets in release mode. Its odd that for some reason, the Debug version of the c++ implementation is at 940MB/s, which is more like the performance i would have expected --- smmap/mman.py | 4 ++-- smmap/test/test_mman.py | 5 ++--- smmap/util.py | 11 +++++++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 8a9066cdc..09fa8fcf9 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -244,7 +244,7 @@ def includes_ofs(self, ofs): :note: always False if the cursor does not point to a valid region""" if self._region is None: return False - return (self.ofs_begin() <= ofs) and (ofs < self.ofs_end()) + return self.ofs_begin() <= ofs < self.ofs_end() def file_size(self): """:return: size of the underlying file""" @@ -326,7 +326,7 @@ def _collect_lru_region(self, size): for regions in self._fdict.itervalues(): for region in regions: # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-1 == 0 and + if (region.client_count()-2 == 0 and (lru_region is None or region.usage_count() < lru_region.usage_count())): lru_region = region lru_list = regions diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index f592fc129..071615269 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -138,15 +138,14 @@ def test_memman_operation(self): st = time() while num_random_accesses: - num_random_accesses += 1 + num_random_accesses -= 1 base_offset = randint(0, fc.size - 1) # precondition assert man.max_mapped_memory_size() >= man.mapped_memory_size() assert man.max_file_handles() >= man.num_file_handles() - assert c.use_region(base_offset, size).is_valid() - assert c.buffer()[:] == data[base_offset:base_offset+size] + assert c.buffer()[:] == data[base_offset:base_offset+c.size()] memory_read += c.size() assert c.includes_ofs(base_offset) diff --git a/smmap/util.py b/smmap/util.py index 55e007554..1aae4f534 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -116,7 +116,7 @@ def __init__(self, path, ofs, size): # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! - self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size - sizeofs), **kwargs) + self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size), **kwargs) if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, size) @@ -125,6 +125,11 @@ def __init__(self, path, ofs, size): os.close(fd) #END close file handle + def __repr__(self): + return "MappedRegion<%i, %i>" % (self._b, self.size()) + + #{ Interface + def buffer(self): """:return: a sliceable buffer which can be used to access the mapped memory""" return self._mf @@ -143,7 +148,7 @@ def ofs_end(self): def includes_ofs(self, ofs): """:return: True if the given offset can be read in our mapped region""" - return (ofs >= self.ofs_begin()) and (ofs <= self.ofs_end()) + return self.ofs_begin() <= ofs < self.ofs_end() def client_count(self): """:return: number of clients currently using this region""" @@ -170,6 +175,8 @@ def buffer(self): return self._mfb #END handle compat layer + #} END interface + class MappedRegionList(list): """List of MappedRegion instances associating a path with a list of regions.""" From 4f3d1ad43740fde43a321abcf59676f8ab9c693f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 16:52:15 +0200 Subject: [PATCH 0153/1392] Applied some optimizations for performance. Python is very easily overwhelmed with plenty of calls, causing too much overhead --- smmap/mman.py | 66 +++++++++++++++++++++++++---------------- smmap/test/test_mman.py | 28 ++++++++++------- smmap/util.py | 19 +++++++----- 3 files changed, 68 insertions(+), 45 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 09fa8fcf9..c13ef6599 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -99,18 +99,30 @@ def use_region(self, offset, size, _is_recursive=False): # END check existing region if need_region: + window_size = man._window_size + # abort on offsets beyond our mapped file's size - currently we are invalid if offset >= self.file_size(): return self # END handle offset too large existing_region = None - for region in self._rlist: - if region.includes_ofs(offset): - existing_region = region - break - #END handle existing region - #END for each existing region + a = self._rlist + lo = 0 + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + ofs = a[mid]._b + if ofs <= offset: + if a[mid].includes_ofs(offset): + existing_region = a[mid] + break + #END have region + lo = mid+1 + else: + hi = mid + #END handle position + #END while bisecting if existing_region is None: left = MemoryWindow(0, 0) @@ -119,20 +131,23 @@ def use_region(self, offset, size, _is_recursive=False): # we want to honor the max memory size, and assure we have anough # memory available - man._collect_lru_region(man.window_size()) + # Save calls ! + if self._manager._memory_size + window_size > self._manager._max_memory_size: + man._collect_lru_region(window_size) + #END handle collection # we assume the list remains sorted by offset insert_pos = 0 - len_regions = len(self._rlist) + len_regions = len(a) if len_regions == 1: - if self._rlist[0].ofs_begin() <= offset: + if a[0]._b <= offset: insert_pos = 1 #END maintain sort else: # find insert position insert_pos = len_regions - for i, region in enumerate(self._rlist): - if region.ofs_begin() > offset: + for i, region in enumerate(a): + if region._b > offset: insert_pos = i break #END if insert position is correct @@ -143,17 +158,17 @@ def use_region(self, offset, size, _is_recursive=False): # possible mapping if insert_pos == 0: if len_regions: - right = MemoryWindow.from_region(self._rlist[insert_pos]) + right = MemoryWindow.from_region(a[insert_pos]) #END adjust right side else: if insert_pos != len_regions: - right = MemoryWindow.from_region(self._rlist[insert_pos]) + right = MemoryWindow.from_region(a[insert_pos]) # END adjust right window - left = MemoryWindow.from_region(self._rlist[insert_pos - 1]) + left = MemoryWindow.from_region(a[insert_pos - 1]) #END adjust surrounding windows - mid.extend_left_to(left, man._window_size) - mid.extend_right_to(right, man._window_size) + mid.extend_left_to(left, window_size) + mid.extend_right_to(right, window_size) mid.align() # it can happen that we align beyond the end of the file @@ -166,7 +181,7 @@ def use_region(self, offset, size, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = MappedRegion(self._rlist.path(), mid.ofs, mid.size) + self._region = MappedRegion(a.path(), mid.ofs, mid.size) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -185,14 +200,14 @@ def use_region(self, offset, size, _is_recursive=False): man._handle_count += 1 man._memory_size += self._region.size() - self._rlist.insert(insert_pos, self._region) + a.insert(insert_pos, self._region) else: self._region = existing_region #END need region handling #END handle acquire region self._region.increment_usage_count() - self._ofs = offset - self._region.ofs_begin() + self._ofs = offset - self._region._b self._size = min(size, self._region.ofs_end() - offset) return self @@ -220,12 +235,12 @@ def is_associated(self): def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor""" - return self._region.ofs_begin() + self._ofs + return self._region._b + self._ofs def ofs_end(self): """:return: offset to one past the last available byte""" # unroll method calls for performance ! - return self._region.ofs_begin() + self._ofs + self._size + return self._region._b + self._ofs + self._size def size(self): """:return: amount of bytes we point to""" @@ -241,10 +256,9 @@ def region_ref(self): def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors current region - :note: always False if the cursor does not point to a valid region""" - if self._region is None: - return False - return self.ofs_begin() <= ofs < self.ofs_end() + :note: cursor must be valid for this to work""" + # unroll methods + return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) def file_size(self): """:return: size of the underlying file""" @@ -327,7 +341,7 @@ def _collect_lru_region(self, size): for region in regions: # check client count - consider that we keep one reference ourselves ! if (region.client_count()-2 == 0 and - (lru_region is None or region.usage_count() < lru_region.usage_count())): + (lru_region is None or region._uc < lru_region._uc)): lru_region = region lru_list = regions # END update lru_region diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 071615269..6f3809979 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -127,9 +127,6 @@ def test_memman_operation(self): # remove mapped regions if we have to assert man.num_file_handles() == 2 - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - # iterate through the windows, verify data contents # this will trigger map collection after a while max_random_accesses = 15000 @@ -137,23 +134,32 @@ def test_memman_operation(self): memory_read = 0 st = time() + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles while num_random_accesses: num_random_accesses -= 1 base_offset = randint(0, fc.size - 1) # precondition - assert man.max_mapped_memory_size() >= man.mapped_memory_size() - assert man.max_file_handles() >= man.num_file_handles() + assert max_mapped_memory_size >= mapped_memory_size() + assert max_file_handles >= num_file_handles() assert c.use_region(base_offset, size).is_valid() - assert c.buffer()[:] == data[base_offset:base_offset+c.size()] - memory_read += c.size() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset+csize] + memory_read += csize - assert c.includes_ofs(base_offset) - assert c.includes_ofs(base_offset+c.size()-1) - assert not c.includes_ofs(base_offset+c.size()) + assert includes_ofs(base_offset) + assert includes_ofs(base_offset+csize-1) + assert not includes_ofs(base_offset+csize) # END while we should do an access elapsed = time() - st mb = 1000 * 1000 - sys.stderr.write("Read %i mb of memory with %i random accesses in %f s(%f mb/s)\n" + sys.stderr.write("Read %i mb of memory with %i random accesses in %fs (%f mb/s)\n" % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() diff --git a/smmap/util.py b/smmap/util.py index 1aae4f534..6a2056b5d 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -3,7 +3,7 @@ import sys import mmap -from mmap import PAGESIZE +from mmap import PAGESIZE, mmap, ACCESS_READ from sys import getrefcount __all__ = [ "align_to_page", "is_64_bit", @@ -48,7 +48,7 @@ def __repr__(self): @classmethod def from_region(cls, region): """:return: new window from a region""" - return cls(region.ofs_begin(), region.size()) + return cls(region._b, region._size) def ofs_end(self): return self.ofs + self.size @@ -83,6 +83,7 @@ class MappedRegion(object): '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_uc', # total amount of usages + '_size', # cached size of our memory map '__weakref__' ] _need_compat_layer = sys.version_info[1] < 6 @@ -100,11 +101,12 @@ def __init__(self, path, ofs, size): allocated the the size automatically adjusted :raise Exception: if no memory can be allocated""" self._b = ofs + self._size = 0 self._uc = 0 fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) try: - kwargs = dict(access=mmap.ACCESS_READ, offset=ofs) + kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size sizeofs = ofs if self._need_compat_layer: @@ -116,7 +118,8 @@ def __init__(self, path, ofs, size): # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! - self._mf = mmap.mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size), **kwargs) + self._mf = mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size), **kwargs) + self._size = len(self._mf) if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, size) @@ -126,7 +129,7 @@ def __init__(self, path, ofs, size): #END close file handle def __repr__(self): - return "MappedRegion<%i, %i>" % (self._b, self.size()) + return "MappedRegion<%i, %i>" % (self._b, self._size) #{ Interface @@ -140,15 +143,15 @@ def ofs_begin(self): def size(self): """:return: total size of the mapped region in bytes""" - return len(self._mf) + return self._size def ofs_end(self): """:return: Absolute offset to one byte beyond the mapping into the file""" - return self._b + self.size() + return self._b + self._size def includes_ofs(self, ofs): """:return: True if the given offset can be read in our mapped region""" - return self.ofs_begin() <= ofs < self.ofs_end() + return self._b <= ofs < self._b + self._size def client_count(self): """:return: number of clients currently using this region""" From 011343f6f43ea868c3efbd00a43757eb82196e1d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 17:46:24 +0200 Subject: [PATCH 0154/1392] Cursor can now take additional flags when opening the file handle for mapping. Renamed stream to buf, as the first item will be a buffer which uses the cursor underneath. We should add a stream for good measure though --- smmap/{stream.py => buf.py} | 0 smmap/mman.py | 10 +++++++--- smmap/test/{test_stream.py => test_buf.py} | 4 ++-- smmap/util.py | 5 +++-- 4 files changed, 12 insertions(+), 7 deletions(-) rename smmap/{stream.py => buf.py} (100%) rename smmap/test/{test_stream.py => test_buf.py} (54%) diff --git a/smmap/stream.py b/smmap/buf.py similarity index 100% rename from smmap/stream.py rename to smmap/buf.py diff --git a/smmap/mman.py b/smmap/mman.py index c13ef6599..57e2d81b5 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -78,10 +78,12 @@ def assign(self, rhs): self._destroy() self._copy_from(rhs) - def use_region(self, offset, size, _is_recursive=False): + def use_region(self, offset, size, flags = 0, _is_recursive=False): """Assure we point to a window which allows access to the given offset into the file :param offset: absolute offset in bytes into the file :param size: amount of bytes to map + :param flags: additional flags to be given to os.open in case a file handle is initially opened + for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. This is not the case if the mapping failed becaues we reached the end of the file :note: The size actually mapped may be smaller than the given size. If that is the case, @@ -106,6 +108,8 @@ def use_region(self, offset, size, _is_recursive=False): return self # END handle offset too large + # bisect to find an existing region. The c++ implementation cannot + # do that as it uses a linked list for regions. existing_region = None a = self._rlist lo = 0 @@ -181,7 +185,7 @@ def use_region(self, offset, size, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = MappedRegion(a.path(), mid.ofs, mid.size) + self._region = MappedRegion(a.path(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -195,7 +199,7 @@ def use_region(self, offset, size, _is_recursive=False): raise #END handle existing recursion man._collect_lru_region(0) - return self.use_region(offset, size, True) + return self.use_region(offset, size, flags, True) #END handle exceptions man._handle_count += 1 diff --git a/smmap/test/test_stream.py b/smmap/test/test_buf.py similarity index 54% rename from smmap/test/test_stream.py rename to smmap/test/test_buf.py index fae928d9a..231abcf5b 100644 --- a/smmap/test/test_stream.py +++ b/smmap/test/test_buf.py @@ -1,7 +1,7 @@ from lib import TestBase -from smmap.stream import * +from smmap.buf import * -class TestStream(TestBase): +class TestBuf(TestBase): def test_basics(self): assert False diff --git a/smmap/util.py b/smmap/util.py index 6c5282cb9..786622d9a 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -93,18 +93,19 @@ class MappedRegion(object): #END handle additional slot - def __init__(self, path, ofs, size): + def __init__(self, path, ofs, size, flags = 0): """Initialize a region, allocate the memory map :param path: path to the file to map :param ofs: **aligned** offset into the file to be mapped :param size: if size is larger then the file on disk, the whole file will be allocated the the size automatically adjusted + :param flags: additional flags to be given when opening the file. :raise Exception: if no memory can be allocated""" self._b = ofs self._size = 0 self._uc = 0 - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) try: kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size From 8670443c3919f5e71e5864717d93eeb7921432ee Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 22:01:10 +0200 Subject: [PATCH 0155/1392] Implemented buffer interface and test to fully proove it. Its not yet fully properly implemented --- smmap/buf.py | 91 +++++++++++++++++++++++++++++++++++++++-- smmap/mman.py | 12 +++++- smmap/test/test_buf.py | 89 +++++++++++++++++++++++++++++++++++++++- smmap/test/test_mman.py | 14 +++++-- 4 files changed, 196 insertions(+), 10 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index bc5e568ea..f7db963ed 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,7 +1,92 @@ -"""Module with a simple stream implementation using the memory manager""" +"""Module with a simple buffer implementation using the memory manager""" +from mman import MemoryCursor -from mman import * +import sys -__all__ = [] +__all__ = ["MappedMemoryBuffer"] + +class MappedMemoryBuffer(object): + """A buffer like object which allows direct byte-wise object and slicing into + memory of a mapped file. The mapping is controlled by an underlying memory manager. + + A buffer, once initialized, stays put on providing access to eactly one path. + A custom interface allows you to change paths mid way, and to optimize + the resource usage. + + Please note that this type is only fully usable if you configure it with the + MappedMemoryManager to use. + + The buffer is relative, that is if you map an offset, index 0 will map to the + first byte at your given offset.""" + __slots__ = '_c' # our cursor + + #{ Configuration + # A subclass must provide an instance of a (usually global) MappedMemoryManager + manager = None + #}END configuration + + def __init__(self, path = None, offset = 0, size = sys.maxint, flags = 0): + """Initalize the instance to operate on the given path if given. + :param path: if not None, the path to the file you want to access + If None, you have call begin_access before using the buffer + :param offset: absolute offset in bytes + :param size: the total size of the mapping. Defaults to the maximum possible size + :param flags: Additional flags to be passed to os.open + :raise ValueError: if the buffer could not achieve a valid state""" + self._c = MemoryCursor(self.manager) + assert self.manager is not None, "Require the cls.manager variable to be set in subclass" + if path and not self.begin_access(path, offset, size, flags): + raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") + # END handle offset + + def __del__(self): + self.end_access() + + def __getitem__(self, i): + c = self._c + if not c.includes_ofs(i): + c.use_region(i, 1) + # END handle region usage + assert c.is_valid() # TODO: remove for performance + return c.buffer()[i] + + def __getslice__(self, i, j): + c = self._c + # fast path, slice fully included - safes a concatenate operation and + # should be the default + if c.ofs_begin() >= i and j < c.ofs_end(): + return c.buffer()[i:j] + raise NotImplementedError() + #{ Interface + + def begin_access(self, path = None, offset = 0, size = sys.maxint, flags = 0): + """Call this before the first use of this instance. The method was already + called by the constructor in case sufficient information was provided. + + For more information no the parameters, see the __init__ method + :param path: if path is empty or None the existing path will be used if possible. + :return: True if the buffer can be used""" + if path and (not self._c.is_associated() or self._c.path() != path): + self._c = self.manager.make_cursor(path) + #END get associated cursor + + # reuse existing cursors if possible + if self._c.is_associated(): + return self._c.use_region(offset, size, flags).is_valid() + return False + + def end_access(self): + """Call this method once you are done using the instance. It is automatically + called on destruction, and should be called just in time to allow system + resources to be freed. + + Once you called end_access, you must call begin access before reusing this instance!""" + self._c.unuse_region() + + def cursor(self): + """:return: the currently set cursor which provides access to the data""" + return self._c + + #}END interface diff --git a/smmap/mman.py b/smmap/mman.py index 57e2d81b5..0fa9ef450 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -226,7 +226,9 @@ def unuse_region(self): def buffer(self): """Return a buffer object which allows access to our memory region from our offset to the window size. Please note that it might be smaller than you requested - :note: You can only obtain a buffer if this instance is_valid() !""" + :note: You can only obtain a buffer if this instance is_valid() ! + :note: buffers should not be cached passed the duration of your access as it will + prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.buffer(), self._ofs, self._size) def is_valid(self): @@ -336,6 +338,7 @@ def _collect_lru_region(self, size): :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region :raise RegionCollectionError: + :return: Amount of freed regions :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): @@ -365,6 +368,8 @@ def _collect_lru_region(self, size): self._handle_count -= 1 #END while there is more memory to free + return num_found + #{ Interface def make_cursor(self, path): """:return: a cursor pointing to the given path. It can be used to map new regions of the file into memory""" @@ -375,6 +380,11 @@ def make_cursor(self, path): # END obtain region for path return MemoryCursor(self, regions) + def collect(self): + """Collect all available free-to-collect mapped regions + :return: Amount of freed handles""" + return self._collect_lru_region(0) + def num_file_handles(self): """:return: amount of file handles in use. Each mapped region uses one file handle""" return self._handle_count diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 231abcf5b..a26c8bb09 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,7 +1,92 @@ -from lib import TestBase +from lib import TestBase, FileCreator +from smmap.mman import MappedMemoryManager from smmap.buf import * +from random import randint +from time import time +import sys + +class TestBuffer(MappedMemoryBuffer): + #{ Configuration + manager = MappedMemoryManager() + #} END configuration + + class TestBuf(TestBase): + def test_basics(self): - assert False + self.failUnlessRaises(AssertionError, MappedMemoryBuffer) # needs subclass + fc = FileCreator(self.k_window_test_size, "buffer_test") + + # invalid paths fail upon construction + self.failUnlessRaises(OSError, TestBuffer, "somefile") # invalid file + self.failUnlessRaises(ValueError, TestBuffer, fc.path, fc.size) # offset too large + + buf = TestBuffer() # can create uninitailized buffers + assert not buf.cursor().is_valid() and not buf.cursor().is_associated() + + # can call end access any time + buf.end_access() + buf.end_access() + + # begin access can revive it, if the offset is suitable + offset = 100 + assert buf.begin_access(fc.path, fc.size) == False + assert buf.begin_access(fc.path, offset) == True + + # empty begin access keeps it valid on the same path, but alters the offset + assert buf.begin_access() == True + assert buf.cursor().is_valid() + + # simple access + data = open(fc.path, 'rb').read() + assert data[offset] == buf[0] + assert data[offset:offset*2] == buf[0:offset] + + # end access makes its cursor invalid + buf.end_access() + assert not buf.cursor().is_valid() + assert buf.cursor().is_associated() # but it remains associated + + # an empty begin access fixes it up again + assert buf.begin_access() == True and buf.cursor().is_valid() + del(buf) # ends access automatically + + man = TestBuffer.manager + assert man.num_file_handles() == 1 + + # PERFORMANCE + # blast away with rnadom access and a full mapping - we don't want to + # exagerate the manager's overhead, but measure the buffer overhead + # We do it once with an optimal setting, and with a worse manager which + # will produce small mappings only ! + max_num_accesses = 5000 + num_accesses_left = max_num_accesses + + for manager in (MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), man): + TestBuffer.manager = manager + st = time() + buf = TestBuffer(fc.path) + assert manager.num_file_handles() == 1 + num_bytes = 0 + fsize = fc.size + while num_accesses_left: + num_accesses_left -= 1 + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + pos = randint(0, fsize) + assert buf[pos] == data[pos] + # END handle num accesses + buf.end_access() + assert manager.num_file_handles() == 1 + assert manager.collect() == 1 + assert manager.num_file_handles() == 0 + elapsed = time() - st + mb = 1000*1000 + sys.stderr.write("Made %i random slices to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (max_num_accesses, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + # END for each manager diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 6f3809979..46a0f50fb 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -75,7 +75,8 @@ def test_memman_operation(self): assert len(data) == fc.size # small windows, a reasonable max memory. Not too many regions at once - man = MappedMemoryManager(fc.size / 100, fc.size / 3, 15) + max_num_handles = 15 + man = MappedMemoryManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) c = man.make_cursor(fc.path) # still empty (more about that is tested in test_memory_manager() @@ -129,7 +130,7 @@ def test_memman_operation(self): # iterate through the windows, verify data contents # this will trigger map collection after a while - max_random_accesses = 15000 + max_random_accesses = 5000 num_random_accesses = max_random_accesses memory_read = 0 st = time() @@ -160,6 +161,11 @@ def test_memman_operation(self): mb = 1000 * 1000 sys.stderr.write("Read %i mb of memory with %i random accesses in %fs (%f mb/s)\n" % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) - + # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 From b00bc5e4c0e8eb47d0bf59c0a2c1f5399f4c8f58 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 23:15:42 +0200 Subject: [PATCH 0156/1392] Finished implementation of buffer including a test which shows the performance should be usable in the real world. Its actually not too bad --- smmap/buf.py | 31 +++++++++++++++++---- smmap/mman.py | 5 +++- smmap/test/test_buf.py | 60 ++++++++++++++++++++++++----------------- smmap/test/test_mman.py | 2 +- 4 files changed, 66 insertions(+), 32 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index f7db963ed..68e7c33b9 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -44,19 +44,40 @@ def __del__(self): def __getitem__(self, i): c = self._c + assert c.is_valid() if not c.includes_ofs(i): c.use_region(i, 1) # END handle region usage - assert c.is_valid() # TODO: remove for performance - return c.buffer()[i] + return c.buffer()[i-c.ofs_begin()] def __getslice__(self, i, j): c = self._c # fast path, slice fully included - safes a concatenate operation and # should be the default - if c.ofs_begin() >= i and j < c.ofs_end(): - return c.buffer()[i:j] - raise NotImplementedError() + assert c.is_valid() + if (c.ofs_begin() <= i) and (j < c.ofs_end()): + b = c.ofs_begin() + return c.buffer()[i-b:j-b] + else: + l = j-i # total length + ofs = i + # keep tokens, and join afterwards. This is faster + # as it can preallocate the total amoint of space needed + # (and its verified the implementation does that) + # Question is whether the list allocation doesn't counteract this, + # but lets see ... + tokens = list() + tappend = tokens.append + + while l: + c.use_region(ofs, l) + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + tappend(d) + #END while there are bytes to read + return ''.join(tokens) + # END fast or slow path #{ Interface def begin_access(self, path = None, offset = 0, size = sys.maxint, flags = 0): diff --git a/smmap/mman.py b/smmap/mman.py index 0fa9ef450..588a63563 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -222,6 +222,8 @@ def unuse_region(self): to unuse the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None + # note: should reset ofs and size, but we spare that for performance. Its not + # allowed to query information if we are not valid ! def buffer(self): """Return a buffer object which allows access to our memory region from our offset @@ -240,7 +242,8 @@ def is_associated(self): return self._rlist is not None def ofs_begin(self): - """:return: offset to the first byte pointed to by our cursor""" + """:return: offset to the first byte pointed to by our cursor + :note: only if is_valid() is True""" return self._region._b + self._ofs def ofs_end(self): diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index a26c8bb09..d7e6d6c96 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -61,32 +61,42 @@ def test_basics(self): # exagerate the manager's overhead, but measure the buffer overhead # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! - max_num_accesses = 5000 - num_accesses_left = max_num_accesses - - for manager in (MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), man): + max_num_accesses = 1000 + for manager, man_id in ( (man, 'optimal'), + (MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), 'worst case')): TestBuffer.manager = manager - st = time() buf = TestBuffer(fc.path) assert manager.num_file_handles() == 1 - num_bytes = 0 - fsize = fc.size - while num_accesses_left: - num_accesses_left -= 1 - ofs_start = randint(0, fsize) - ofs_end = randint(ofs_start, fsize) - d = buf[ofs_start:ofs_end] - assert len(d) == ofs_end - ofs_start - assert d == data[ofs_start:ofs_end] - num_bytes += len(d) - pos = randint(0, fsize) - assert buf[pos] == data[pos] - # END handle num accesses - buf.end_access() - assert manager.num_file_handles() == 1 - assert manager.collect() == 1 - assert manager.num_file_handles() == 0 - elapsed = time() - st - mb = 1000*1000 - sys.stderr.write("Made %i random slices to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (max_num_accesses, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + for access_mode in range(2): # single, multi + num_accesses_left = max_num_accesses + num_bytes = 0 + fsize = fc.size + + st = time() + buf.begin_access() + while num_accesses_left: + num_accesses_left -= 1 + if access_mode: # multi + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + else: + pos = randint(0, fsize) + assert buf[pos] == data[pos] + num_bytes += 1 + #END handle mode + # END handle num accesses + + buf.end_access() + assert manager.num_file_handles() + assert manager.collect() + assert manager.num_file_handles() == 0 + elapsed = time() - st + mb = float(1000*1000) + mode_str = (access_mode and "slice") or "single byte" + sys.stderr.write("%s: Made %i random %s accesses to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (man_id, max_num_accesses, mode_str, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + # END handle access mode # END for each manager diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 46a0f50fb..b1c8f68eb 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -158,7 +158,7 @@ def test_memman_operation(self): assert not includes_ofs(base_offset+csize) # END while we should do an access elapsed = time() - st - mb = 1000 * 1000 + mb = float(1000 * 1000) sys.stderr.write("Read %i mb of memory with %i random accesses in %fs (%f mb/s)\n" % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) From 68fba826cdfd0de532ccf14c9ad94bf035a36e1b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 23:22:46 +0200 Subject: [PATCH 0157/1392] Optimized __getslice__ implementation a bit --- smmap/buf.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 68e7c33b9..d1be1b65a 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -61,22 +61,17 @@ def __getslice__(self, i, j): else: l = j-i # total length ofs = i - # keep tokens, and join afterwards. This is faster - # as it can preallocate the total amoint of space needed - # (and its verified the implementation does that) - # Question is whether the list allocation doesn't counteract this, - # but lets see ... - tokens = list() - tappend = tokens.append - + # Keeping tokens in a list could possible be faster, but the list + # overhead outweighs the benefits (tested) ! + md = str() while l: c.use_region(ofs, l) d = c.buffer()[:l] ofs += len(d) l -= len(d) - tappend(d) + md += d #END while there are bytes to read - return ''.join(tokens) + return md # END fast or slow path #{ Interface From c30e930bf06edc8f60ce3311d13f6a61de8ac0ce Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 9 Jun 2011 23:57:55 +0200 Subject: [PATCH 0158/1392] Changed buffer implementation to use a cursor right away instead of taking a path and a manager. This makes it much more flexible, as it doesn't have to care about the manager anymore, making it easier to use and making clear that it is meant for use with a mapped memory manager implementation. --- smmap/buf.py | 44 ++++++++++++++++-------------------------- smmap/test/test_buf.py | 36 +++++++++++++++++----------------- 2 files changed, 35 insertions(+), 45 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index d1be1b65a..b4f581316 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -7,35 +7,23 @@ class MappedMemoryBuffer(object): """A buffer like object which allows direct byte-wise object and slicing into - memory of a mapped file. The mapping is controlled by an underlying memory manager. - - A buffer, once initialized, stays put on providing access to eactly one path. - A custom interface allows you to change paths mid way, and to optimize - the resource usage. - - Please note that this type is only fully usable if you configure it with the - MappedMemoryManager to use. + memory of a mapped file. The mapping is controlled by the provided cursor. The buffer is relative, that is if you map an offset, index 0 will map to the - first byte at your given offset.""" + first byte at the offset you used during initialization or begin_access""" __slots__ = '_c' # our cursor - #{ Configuration - # A subclass must provide an instance of a (usually global) MappedMemoryManager - manager = None - #}END configuration - def __init__(self, path = None, offset = 0, size = sys.maxint, flags = 0): - """Initalize the instance to operate on the given path if given. - :param path: if not None, the path to the file you want to access - If None, you have call begin_access before using the buffer + def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + """Initalize the instance to operate on the given cursor. + :param cursor: if not None, the associated cursor to the file you want to access + If None, you have call begin_access before using the buffer and provide a cursor :param offset: absolute offset in bytes :param size: the total size of the mapping. Defaults to the maximum possible size :param flags: Additional flags to be passed to os.open :raise ValueError: if the buffer could not achieve a valid state""" - self._c = MemoryCursor(self.manager) - assert self.manager is not None, "Require the cls.manager variable to be set in subclass" - if path and not self.begin_access(path, offset, size, flags): + self._c = cursor + if cursor and not self.begin_access(cursor, offset, size, flags): raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") # END handle offset @@ -75,19 +63,19 @@ def __getslice__(self, i, j): # END fast or slow path #{ Interface - def begin_access(self, path = None, offset = 0, size = sys.maxint, flags = 0): + def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): """Call this before the first use of this instance. The method was already called by the constructor in case sufficient information was provided. For more information no the parameters, see the __init__ method - :param path: if path is empty or None the existing path will be used if possible. + :param path: if cursor is None the existing one will be used. :return: True if the buffer can be used""" - if path and (not self._c.is_associated() or self._c.path() != path): - self._c = self.manager.make_cursor(path) - #END get associated cursor + if cursor: + self._c = cursor + #END update our cursor # reuse existing cursors if possible - if self._c.is_associated(): + if self._c is not None and self._c.is_associated(): return self._c.use_region(offset, size, flags).is_valid() return False @@ -97,7 +85,9 @@ def end_access(self): resources to be freed. Once you called end_access, you must call begin access before reusing this instance!""" - self._c.unuse_region() + if self._c is not None: + self._c.unuse_region() + #END unuse region def cursor(self): """:return: the currently set cursor which provides access to the data""" diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d7e6d6c96..6192897bd 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -7,24 +7,24 @@ from time import time import sys -class TestBuffer(MappedMemoryBuffer): - #{ Configuration - manager = MappedMemoryManager() - #} END configuration - + +man_optimal = MappedMemoryManager() +man_worst_case = MappedMemoryManager( window_size=TestBase.k_window_test_size/100, + max_memory_size=TestBase.k_window_test_size/3, + max_open_handles=15) class TestBuf(TestBase): def test_basics(self): - self.failUnlessRaises(AssertionError, MappedMemoryBuffer) # needs subclass fc = FileCreator(self.k_window_test_size, "buffer_test") # invalid paths fail upon construction - self.failUnlessRaises(OSError, TestBuffer, "somefile") # invalid file - self.failUnlessRaises(ValueError, TestBuffer, fc.path, fc.size) # offset too large + c = man_optimal.make_cursor(fc.path) + self.failUnlessRaises(ValueError, MappedMemoryBuffer, type(c)()) # invalid cursor + self.failUnlessRaises(ValueError, MappedMemoryBuffer, c, fc.size) # offset too large - buf = TestBuffer() # can create uninitailized buffers - assert not buf.cursor().is_valid() and not buf.cursor().is_associated() + buf = MappedMemoryBuffer() # can create uninitailized buffers + assert buf.cursor() is None # can call end access any time buf.end_access() @@ -32,8 +32,9 @@ def test_basics(self): # begin access can revive it, if the offset is suitable offset = 100 - assert buf.begin_access(fc.path, fc.size) == False - assert buf.begin_access(fc.path, offset) == True + assert buf.begin_access(c, fc.size) == False + assert buf.begin_access(c, offset) == True + assert buf.cursor().is_valid() # empty begin access keeps it valid on the same path, but alters the offset assert buf.begin_access() == True @@ -52,9 +53,9 @@ def test_basics(self): # an empty begin access fixes it up again assert buf.begin_access() == True and buf.cursor().is_valid() del(buf) # ends access automatically + del(c) - man = TestBuffer.manager - assert man.num_file_handles() == 1 + assert man_optimal.num_file_handles() == 1 # PERFORMANCE # blast away with rnadom access and a full mapping - we don't want to @@ -62,10 +63,9 @@ def test_basics(self): # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! max_num_accesses = 1000 - for manager, man_id in ( (man, 'optimal'), - (MappedMemoryManager(window_size=fc.size/100, max_memory_size=fc.size/3, max_open_handles=15), 'worst case')): - TestBuffer.manager = manager - buf = TestBuffer(fc.path) + for manager, man_id in ( (man_optimal, 'optimal'), + (man_worst_case, 'worst case')): + buf = MappedMemoryBuffer(manager.make_cursor(fc.path)) assert manager.num_file_handles() == 1 for access_mode in range(2): # single, multi num_accesses_left = max_num_accesses From aafc980e9cbb1b2ed08d374300e9916538136cc7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 00:02:00 +0200 Subject: [PATCH 0159/1392] Added indirection level to internally used types to allow others to exchange them with their own implementations. --- smmap/mman.py | 25 +++++++++++++++++-------- smmap/test/test_buf.py | 2 +- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 588a63563..d8626d065 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -29,6 +29,11 @@ class MemoryCursor(object): '_size' # maximum size we should provide ) + #{ Configuration + MemoryWindowCls = MemoryWindow + MappedRegionCls = MappedRegion + #} END configuration + def __init__(self, manager = None, regions = None): self._manager = manager self._rlist = regions @@ -129,9 +134,9 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): #END while bisecting if existing_region is None: - left = MemoryWindow(0, 0) - mid = MemoryWindow(offset, size) - right = MemoryWindow(self.file_size(), 0) + left = self.MemoryWindowCls(0, 0) + mid = self.MemoryWindowCls(offset, size) + right = self.MemoryWindowCls(self.file_size(), 0) # we want to honor the max memory size, and assure we have anough # memory available @@ -162,13 +167,13 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): # possible mapping if insert_pos == 0: if len_regions: - right = MemoryWindow.from_region(a[insert_pos]) + right = self.MemoryWindowCls.from_region(a[insert_pos]) #END adjust right side else: if insert_pos != len_regions: - right = MemoryWindow.from_region(a[insert_pos]) + right = self.MemoryWindowCls.from_region(a[insert_pos]) # END adjust right window - left = MemoryWindow.from_region(a[insert_pos - 1]) + left = self.MemoryWindowCls.from_region(a[insert_pos - 1]) #END adjust surrounding windows mid.extend_left_to(left, window_size) @@ -185,7 +190,7 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = MappedRegion(a.path(), mid.ofs, mid.size, flags) + self._region = self.MappedRegionCls(a.path(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -302,6 +307,10 @@ class MappedMemoryManager(object): '_handle_count', # amount of currently allocated file handles ] + #{ Configuration + MappedRegionListCls = MappedRegionList + #} END configuration + _MB_in_bytes = 1024 * 1024 def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): @@ -378,7 +387,7 @@ def make_cursor(self, path): """:return: a cursor pointing to the given path. It can be used to map new regions of the file into memory""" regions = self._fdict.get(path) if regions is None: - regions = MappedRegionList(path) + regions = self.MappedRegionListCls(path) self._fdict[path] = regions # END obtain region for path return MemoryCursor(self, regions) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 6192897bd..ae1a174d5 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -62,7 +62,7 @@ def test_basics(self): # exagerate the manager's overhead, but measure the buffer overhead # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! - max_num_accesses = 1000 + max_num_accesses = 400 for manager, man_id in ( (man_optimal, 'optimal'), (man_worst_case, 'worst case')): buf = MappedMemoryBuffer(manager.make_cursor(fc.path)) From 8d64e74ed80f2818acad652a69615708f8f61104 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 00:38:48 +0200 Subject: [PATCH 0160/1392] Added very special purpose method to free memory maps. The whole reason for this is to make the windows test work after all \! --- smmap/mman.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/smmap/mman.py b/smmap/mman.py index d8626d065..44d985e28 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -426,3 +426,32 @@ def page_size(self): return PAGESIZE #} END interface + + #{ Special Purpose Interface + + def force_map_handle_removal_win(self, base_path): + """ONLY AVAILABLE ON WINDOWS + On windows removing files is not allowed if anybody still has it opened. + If this process is ourselves, and if the whole process uses this memory + manager (as far as the parent framework is concerned) we can enforce + closing all memory maps whose path matches the given base path to + allow the respective operation after all. + The respective system must NOT access the closed memory regions anymore ! + This really may only be used if you know that the items which keep + the cursors alive will not be using it anymore. They need to be recreated ! + :return: Amount of closed handles + :note: does nothing on non-windows platforms""" + if sys.platform != 'win32': + return + #END early bailout + + num_closed = 0 + for path, rlist in self._fdict.iteritems(): + if path.startswith(base_path): + for region in rlist: + region._mf.close() + num_closed += 1 + #END path matches + #END for each path + return num_closed + #} END special purpose interface From 4ad22313d8764bcc3645ba8d8ea3ff6f50bba7d1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 09:42:55 +0200 Subject: [PATCH 0161/1392] System can now deal with file descriptors as input, but it still requires some more testing. Also fds need to remain open to be usable for new mapped regions --- smmap/mman.py | 41 ++++++++++++++++++++++++++++++++--------- smmap/test/test_util.py | 18 ++++++++++++------ smmap/util.py | 29 +++++++++++++++++++---------- 3 files changed, 63 insertions(+), 25 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 44d985e28..79f2de896 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -54,7 +54,7 @@ def _destroy(self): num_clients = self._rlist.client_count() - 2 if num_clients == 0 and len(self._rlist) == 0: # Free all resources associated with the mapped file - self._manager._fdict.pop(self._rlist.path()) + self._manager._fdict.pop(self._rlist.path_or_fd()) #END remove regions list from manager #END handle regions @@ -190,7 +190,7 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = self.MappedRegionCls(a.path(), mid.ofs, mid.size, flags) + self._region = self.MappedRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -278,9 +278,26 @@ def file_size(self): """:return: size of the underlying file""" return self._rlist.file_size() + def path_or_fd(self): + """:return: path or file decriptor of the underlying mapped file""" + return self._rlist.path_or_fd() + def path(self): - """:return: path of the underlying mapped file""" - return self._rlist.path() + """:return: path of the underlying mapped file + :raise ValueError: if attached path is not a path""" + if isinstance(self._rlist.path_or_fd(), int): + raise ValueError("Path queried although mapping was applied to a file descriptor") + # END handle type + return self._rlist.path_or_fd() + + def fd(self): + """:return: file descriptor used to create the underlying mapping. + :note: it is not required to be valid anymore + :raise ValueError: if the mapping was not created by a file descriptor""" + if isinstance(self._rlist.path_or_fd(), basestring): + return ValueError("File descriptor queried although mapping was generated from path") + #END handle type + return self._rlist.path_or_fd() #} END interface @@ -383,12 +400,18 @@ def _collect_lru_region(self, size): return num_found #{ Interface - def make_cursor(self, path): - """:return: a cursor pointing to the given path. It can be used to map new regions of the file into memory""" - regions = self._fdict.get(path) + def make_cursor(self, path_or_fd): + """:return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory + :note: if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open.""" + regions = self._fdict.get(path_or_fd) if regions is None: - regions = self.MappedRegionListCls(path) - self._fdict[path] = regions + regions = self.MappedRegionListCls(path_or_fd) + self._fdict[path_or_fd] = regions # END obtain region for path return MemoryCursor(self, regions) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 136d99102..a5478cd7c 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -2,6 +2,7 @@ from smmap.util import * +import os import sys class TestMMan(TestBase): @@ -86,13 +87,18 @@ def test_region(self): def test_region_list(self): fc = FileCreator(100, "sample_file") - ml = MappedRegionList(fc.path) - assert ml.client_count() == 1 - - assert len(ml) == 0 - assert ml.path() == fc.path - assert ml.file_size() == fc.size + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + ml = MappedRegionList(item) + + assert ml.client_count() == 1 + + assert len(ml) == 0 + assert ml.path_or_fd() == item + assert ml.file_size() == fc.size + #END handle input + os.close(fd) def test_util(self): assert isinstance(is_64_bit(), bool) # just call it diff --git a/smmap/util.py b/smmap/util.py index 786622d9a..f3bb58b33 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -93,9 +93,9 @@ class MappedRegion(object): #END handle additional slot - def __init__(self, path, ofs, size, flags = 0): + def __init__(self, path_or_fd, ofs, size, flags = 0): """Initialize a region, allocate the memory map - :param path: path to the file to map + :param path_or_fd: path to the file to map, or the opened file descriptor :param ofs: **aligned** offset into the file to be mapped :param size: if size is larger then the file on disk, the whole file will be allocated the the size automatically adjusted @@ -105,7 +105,12 @@ def __init__(self, path, ofs, size, flags = 0): self._size = 0 self._uc = 0 - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + if isinstance(path_or_fd, int): + fd = path_or_fd + else: + fd = os.open(path_or_fd, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + #END handle fd + try: kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size @@ -189,29 +194,33 @@ def includes_ofs(self, ofs): class MappedRegionList(list): """List of MappedRegion instances associating a path with a list of regions.""" __slots__ = ( - '_path', # path which is mapped by all our regions + '_path_or_fd', # path or file descriptor which is mapped by all our regions '_file_size' # total size of the file we map ) def __new__(cls, path): return super(MappedRegionList, cls).__new__(cls) - def __init__(self, path): - self._path = path + def __init__(self, path_or_fd): + self._path_or_fd = path_or_fd self._file_size = None def client_count(self): """:return: amount of clients which hold a reference to this instance""" return getrefcount(self)-3 - def path(self): - """:return: path to file whose regions we manage""" - return self._path + def path_or_fd(self): + """:return: path or file descriptor we are attached to""" + return self._path_or_fd def file_size(self): """:return: size of file we manager""" if self._file_size is None: - self._file_size = os.stat(self._path).st_size + if isinstance(self._path_or_fd, basestring): + self._file_size = os.stat(self._path_or_fd).st_size + else: + self._file_size = os.fstat(self._path_or_fd).st_size + #END handle path type #END update file size return self._file_size From a8a5e10835d71bb99993744777a06fc5573c892c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 09:53:10 +0200 Subject: [PATCH 0162/1392] Added tests for the fd case. It shows that this is measurably faster than the string path version because there is less system overhead. Good to know actally --- smmap/mman.py | 4 +- smmap/test/test_buf.py | 80 +++++++++------- smmap/test/test_mman.py | 207 +++++++++++++++++++++------------------- smmap/util.py | 4 +- 4 files changed, 157 insertions(+), 138 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 79f2de896..b15b9af7c 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -407,7 +407,9 @@ def make_cursor(self, path_or_fd): but may be closed afterwards. To refer to the same file, you may reuse your existing file descriptor, but keep in mind that new windows can only be mapped as long as it stays valid. This is why the using actual file paths - are preferred unless you plan to keep the file descriptor open.""" + are preferred unless you plan to keep the file descriptor open. + :note: Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) if regions is None: regions = self.MappedRegionListCls(path_or_fd) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index ae1a174d5..efc1da60c 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -6,6 +6,7 @@ from random import randint from time import time import sys +import os man_optimal = MappedMemoryManager() @@ -63,40 +64,45 @@ def test_basics(self): # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! max_num_accesses = 400 - for manager, man_id in ( (man_optimal, 'optimal'), - (man_worst_case, 'worst case')): - buf = MappedMemoryBuffer(manager.make_cursor(fc.path)) - assert manager.num_file_handles() == 1 - for access_mode in range(2): # single, multi - num_accesses_left = max_num_accesses - num_bytes = 0 - fsize = fc.size - - st = time() - buf.begin_access() - while num_accesses_left: - num_accesses_left -= 1 - if access_mode: # multi - ofs_start = randint(0, fsize) - ofs_end = randint(ofs_start, fsize) - d = buf[ofs_start:ofs_end] - assert len(d) == ofs_end - ofs_start - assert d == data[ofs_start:ofs_end] - num_bytes += len(d) - else: - pos = randint(0, fsize) - assert buf[pos] == data[pos] - num_bytes += 1 - #END handle mode - # END handle num accesses - - buf.end_access() - assert manager.num_file_handles() - assert manager.collect() - assert manager.num_file_handles() == 0 - elapsed = time() - st - mb = float(1000*1000) - mode_str = (access_mode and "slice") or "single byte" - sys.stderr.write("%s: Made %i random %s accesses to buffer reading a total of %f mb in %f s (%f mb/s)\n" % (man_id, max_num_accesses, mode_str, num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) - # END handle access mode - # END for each manager + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + for manager, man_id in ( (man_optimal, 'optimal'), + (man_worst_case, 'worst case')): + buf = MappedMemoryBuffer(manager.make_cursor(item)) + assert manager.num_file_handles() == 1 + for access_mode in range(2): # single, multi + num_accesses_left = max_num_accesses + num_bytes = 0 + fsize = fc.size + + st = time() + buf.begin_access() + while num_accesses_left: + num_accesses_left -= 1 + if access_mode: # multi + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + else: + pos = randint(0, fsize) + assert buf[pos] == data[pos] + num_bytes += 1 + #END handle mode + # END handle num accesses + + buf.end_access() + assert manager.num_file_handles() + assert manager.collect() + assert manager.num_file_handles() == 0 + elapsed = time() - st + mb = float(1000*1000) + mode_str = (access_mode and "slice") or "single byte" + sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" + % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + # END handle access mode + # END for each manager + # END for each input + os.close(fd) diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index b1c8f68eb..57d78d504 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -7,6 +7,7 @@ from random import randint from time import time +import os import sys from copy import copy @@ -62,110 +63,118 @@ def test_memory_manager(self): # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") - c = man.make_cursor(fc.path) - assert c.use_region(10, 10).is_valid() - assert c.ofs_begin() == 10 - assert c.size() == 10 - assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + c = man.make_cursor(item) + assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + #END for each input + os.close(fd) def test_memman_operation(self): # test more access, force it to actually unmap regions fc = FileCreator(self.k_window_test_size, "manager_operation_test") data = open(fc.path, 'rb').read() - assert len(data) == fc.size - - # small windows, a reasonable max memory. Not too many regions at once - max_num_handles = 15 - man = MappedMemoryManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) - c = man.make_cursor(fc.path) - - # still empty (more about that is tested in test_memory_manager() - assert man.num_open_files() == 0 - assert man.mapped_memory_size() == 0 - - base_offset = 5000 - size = man.window_size() / 2 - assert c.use_region(base_offset, size).is_valid() - rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us - - assert man.num_open_files() == 1 - assert man.num_file_handles() == 1 - assert man.mapped_memory_size() == rr().size() - assert c.size() == size - assert c.ofs_begin() == base_offset - assert rr().ofs_begin() == 0 # it was aligned and expanded - assert rr().size() == align_to_page(man.window_size(), True) # but isn't larger than the max window (aligned) - - assert c.buffer()[:] == data[base_offset:base_offset+size] - - # obtain second window, which spans the first part of the file - it is a still the same window - assert c.use_region(0, size-10).is_valid() - assert c.region_ref()() == rr() - assert man.num_file_handles() == 1 - assert c.size() == size-10 - assert c.ofs_begin() == 0 - assert c.buffer()[:] == data[:size-10] - - # map some part at the end, our requested size cannot be kept - overshoot = 4000 - base_offset = fc.size - size + overshoot - assert c.use_region(base_offset, size).is_valid() - assert man.num_file_handles() == 2 - assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager - rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset+size] - - # unising a region makes the cursor invalid - c.unuse_region() - assert not c.is_valid() - # but doesn't change anything regarding the handle count - we cache it and only - # remove mapped regions if we have to - assert man.num_file_handles() == 2 - - # iterate through the windows, verify data contents - # this will trigger map collection after a while - max_random_accesses = 5000 - num_random_accesses = max_random_accesses - memory_read = 0 - st = time() - - # cache everything to get some more performance - includes_ofs = c.includes_ofs - max_mapped_memory_size = man.max_mapped_memory_size() - max_file_handles = man.max_file_handles() - mapped_memory_size = man.mapped_memory_size - num_file_handles = man.num_file_handles - while num_random_accesses: - num_random_accesses -= 1 - base_offset = randint(0, fc.size - 1) + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + max_num_handles = 15 + man = MappedMemoryManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) + c = man.make_cursor(item) - # precondition - assert max_mapped_memory_size >= mapped_memory_size() - assert max_file_handles >= num_file_handles() + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + size = man.window_size() / 2 assert c.use_region(base_offset, size).is_valid() - csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset+csize] - memory_read += csize + rr = c.region_ref() + assert rr().client_count() == 2 # the manager and the cursor and us - assert includes_ofs(base_offset) - assert includes_ofs(base_offset+csize-1) - assert not includes_ofs(base_offset+csize) - # END while we should do an access - elapsed = time() - st - mb = float(1000 * 1000) - sys.stderr.write("Read %i mb of memory with %i random accesses in %fs (%f mb/s)\n" - % (memory_read/mb, max_random_accesses, elapsed, (memory_read/mb)/elapsed)) - - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - - # collection - it should be able to collect all - assert man.num_file_handles() - assert man.collect() - assert man.num_file_handles() == 0 + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr().size() + assert c.size() == size + assert c.ofs_begin() == base_offset + assert rr().ofs_begin() == 0 # it was aligned and expanded + assert rr().size() == align_to_page(man.window_size(), True) # but isn't larger than the max window (aligned) + + assert c.buffer()[:] == data[base_offset:base_offset+size] + + # obtain second window, which spans the first part of the file - it is a still the same window + assert c.use_region(0, size-10).is_valid() + assert c.region_ref()() == rr() + assert man.num_file_handles() == 1 + assert c.size() == size-10 + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:size-10] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - size + overshoot + assert c.use_region(base_offset, size).is_valid() + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager + rr = c.region_ref() + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset+size] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 5000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles + while num_random_accesses: + num_random_accesses -= 1 + base_offset = randint(0, fc.size - 1) + + # precondition + assert max_mapped_memory_size >= mapped_memory_size() + assert max_file_handles >= num_file_handles() + assert c.use_region(base_offset, size).is_valid() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset+csize] + memory_read += csize + + assert includes_ofs(base_offset) + assert includes_ofs(base_offset+csize-1) + assert not includes_ofs(base_offset+csize) + # END while we should do an access + elapsed = time() - st + mb = float(1000 * 1000) + sys.stderr.write("Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 + #END for each item + os.close(fd) diff --git a/smmap/util.py b/smmap/util.py index f3bb58b33..2ba7a1d29 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -131,7 +131,9 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): self._mfb = buffer(self._mf, ofs, size) #END handle buffer wrapping finally: - os.close(fd) + if isinstance(path_or_fd, basestring): + os.close(fd) + #END only close it if we opened it #END close file handle def __repr__(self): From 9f16040fb4cde025875e28b01d70cfba11c1d773 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 10:42:32 +0200 Subject: [PATCH 0163/1392] Fixed some mapping issues on windows. Fixed some tests to deal with the very different granularity --- smmap/mman.py | 5 ----- smmap/test/test_buf.py | 2 +- smmap/test/test_mman.py | 7 +++---- smmap/test/test_util.py | 17 ++++++++++++----- smmap/util.py | 16 ++++++++-------- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index b15b9af7c..8e50a1445 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -4,7 +4,6 @@ MappedRegion, MappedRegionList, is_64_bit, - PAGESIZE ) from exc import RegionCollectionError @@ -446,10 +445,6 @@ def max_mapped_memory_size(self): """:return: maximum amount of memory we may allocate""" return self._max_memory_size - def page_size(self): - """:return: size of a single memory page in bytes""" - return PAGESIZE - #} END interface #{ Special Purpose Interface diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index efc1da60c..c772e4999 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -97,7 +97,7 @@ def test_basics(self): assert manager.num_file_handles() assert manager.collect() assert manager.num_file_handles() == 0 - elapsed = time() - st + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows mb = float(1000*1000) mode_str = (access_mode and "slice") or "single byte" sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 57d78d504..e220f8bb2 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -2,7 +2,7 @@ from smmap.mman import * from smmap.mman import MemoryCursor -from smmap.util import PAGESIZE, align_to_page +from smmap.util import align_to_mmap from smmap.exc import RegionCollectionError from random import randint @@ -52,7 +52,6 @@ def test_memory_manager(self): assert man.window_size() > 0 assert man.mapped_memory_size() == 0 assert man.max_mapped_memory_size() > 0 - assert man.page_size() == PAGESIZE # collection doesn't raise in 'any' mode man._collect_lru_region(0) @@ -102,7 +101,7 @@ def test_memman_operation(self): assert c.size() == size assert c.ofs_begin() == base_offset assert rr().ofs_begin() == 0 # it was aligned and expanded - assert rr().size() == align_to_page(man.window_size(), True) # but isn't larger than the max window (aligned) + assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) assert c.buffer()[:] == data[base_offset:base_offset+size] @@ -164,7 +163,7 @@ def test_memman_operation(self): assert includes_ofs(base_offset+csize-1) assert not includes_ofs(base_offset+csize) # END while we should do an access - elapsed = time() - st + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows mb = float(1000 * 1000) sys.stderr.write("Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" % (memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index a5478cd7c..7caa427ba 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,6 +1,7 @@ from lib import TestBase, FileCreator from smmap.util import * +from mmap import ALLOCATIONGRANULARITY import os import sys @@ -50,12 +51,12 @@ def test_window(self): assert wr.ofs == wc2.ofs_end() wc.align() - assert wc.ofs == 0 and wc.size == PAGESIZE*2 + assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) def test_region(self): fc = FileCreator(self.k_window_test_size, "window_test") half_size = fc.size / 2 - rofs = align_to_page(4200, False) + rofs = align_to_mmap(4200, False) rfull = MappedRegion(fc.path, 0, fc.size) rhalfofs = MappedRegion(fc.path, rofs, fc.size) rhalfsize = MappedRegion(fc.path, 0, half_size) @@ -69,7 +70,13 @@ def test_region(self): assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) - assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + # with the values we have, this test only works on windows where an alignment + # size of 4096 is assumed. + if sys.platform == 'win32': + assert rhalfofs.includes_ofs(rofs) and rhalfofs.includes_ofs(0) + else: + assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + #END handle platforms # auto-refcount assert rfull.client_count() == 1 @@ -102,6 +109,6 @@ def test_region_list(self): def test_util(self): assert isinstance(is_64_bit(), bool) # just call it - assert align_to_page(1, False) == 0 - assert align_to_page(1, True) == PAGESIZE + assert align_to_mmap(1, False) == 0 + assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY diff --git a/smmap/util.py b/smmap/util.py index 2ba7a1d29..e1f786538 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -3,22 +3,22 @@ import sys import mmap -from mmap import PAGESIZE, mmap, ACCESS_READ +from mmap import ALLOCATIONGRANULARITY, mmap, ACCESS_READ from sys import getrefcount -__all__ = [ "align_to_page", "is_64_bit", - "MemoryWindow", "MappedRegion", "MappedRegionList", "PAGESIZE"] +__all__ = [ "align_to_mmap", "is_64_bit", + "MemoryWindow", "MappedRegion", "MappedRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities -def align_to_page(num, round_up): +def align_to_mmap(num, round_up): """Align the given integer number to the closest page offset, which usually is 4096 bytes. :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" - res = (num / PAGESIZE) * PAGESIZE; + res = (num / ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; if round_up and (res != num): - res += PAGESIZE; + res += ALLOCATIONGRANULARITY #END handle size return res; @@ -55,10 +55,10 @@ def ofs_end(self): def align(self): """Assures the previous window area is contained in the new one""" - nofs = align_to_page(self.ofs, 0) + nofs = align_to_mmap(self.ofs, 0) self.size += self.ofs - nofs # keep size constant self.ofs = nofs - self.size = align_to_page(self.size, 1) + self.size = align_to_mmap(self.size, 1) def extend_left_to(self, window, max_size): """Adjust the offset to start where the given window on our left ends if possible, From 4466476cf576cf5936a11524e67345a80e2ec5a9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 10:48:28 +0200 Subject: [PATCH 0164/1392] Fixed missing ALLOCATIONGRANULARIY in python <2.6. Python is as unportable as ever across versions with simple functionality --- smmap/test/test_util.py | 1 - smmap/util.py | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 7caa427ba..4043cd83f 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,7 +1,6 @@ from lib import TestBase, FileCreator from smmap.util import * -from mmap import ALLOCATIONGRANULARITY import os import sys diff --git a/smmap/util.py b/smmap/util.py index e1f786538..667777861 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -3,7 +3,15 @@ import sys import mmap -from mmap import ALLOCATIONGRANULARITY, mmap, ACCESS_READ +from mmap import mmap, ACCESS_READ +try: + from mmap import ALLOCATIONGRANULARITY +except ImportError: + # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly + # useful for aligning the offset. The offset argument doesn't exist there though + from mmap import PAGESIZE as ALLOCATIONGRANULARITY +#END handle pythons missing quality assurance + from sys import getrefcount __all__ = [ "align_to_mmap", "is_64_bit", From 064aa81076b8b2e08209aa6525ca22065703b41d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 12:35:32 +0200 Subject: [PATCH 0165/1392] Implemented __len__ method in buffer, including small test. This has its caveats, but should be fine for responsible clients --- smmap/buf.py | 26 ++++++++++++++++++++++++-- smmap/test/test_buf.py | 5 ++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index b4f581316..4fc78ea6f 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -11,7 +11,10 @@ class MappedMemoryBuffer(object): The buffer is relative, that is if you map an offset, index 0 will map to the first byte at the offset you used during initialization or begin_access""" - __slots__ = '_c' # our cursor + __slots__ = ( + '_c', # our cursor + '_size', # our supposed size + ) def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): @@ -20,6 +23,10 @@ def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): If None, you have call begin_access before using the buffer and provide a cursor :param offset: absolute offset in bytes :param size: the total size of the mapping. Defaults to the maximum possible size + From that point on, the __len__ of the buffer will be the given size or the file size. + If the size is larger than the mappable area, you can only access the actually available + area, although the length of the buffer is reported to be your given size. + Hence it is in your own interest to provide a proper size ! :param flags: Additional flags to be passed to os.open :raise ValueError: if the buffer could not achieve a valid state""" self._c = cursor @@ -30,6 +37,9 @@ def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): def __del__(self): self.end_access() + def __len__(self): + return self._size + def __getitem__(self, i): c = self._c assert c.is_valid() @@ -76,7 +86,18 @@ def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): # reuse existing cursors if possible if self._c is not None and self._c.is_associated(): - return self._c.use_region(offset, size, flags).is_valid() + res = self._c.use_region(offset, size, flags).is_valid() + if res: + # if given size is too large or default, we computer a proper size + # If its smaller, we assume the combination between offset and size + # as chosen by the user is correct and use it ! + # If not, the user is in trouble. + if size > self._c.file_size(): + size = self._c.file_size() - offset + #END handle size + self._size = size + #END set size + return res return False def end_access(self): @@ -85,6 +106,7 @@ def end_access(self): resources to be freed. Once you called end_access, you must call begin access before reusing this instance!""" + self._size = 0 if self._c is not None: self._c.unuse_region() #END unuse region diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index c772e4999..48aeabb44 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -30,15 +30,18 @@ def test_basics(self): # can call end access any time buf.end_access() buf.end_access() + assert len(buf) == 0 # begin access can revive it, if the offset is suitable offset = 100 assert buf.begin_access(c, fc.size) == False assert buf.begin_access(c, offset) == True + assert len(buf) == fc.size - offset assert buf.cursor().is_valid() # empty begin access keeps it valid on the same path, but alters the offset assert buf.begin_access() == True + assert len(buf) == fc.size assert buf.cursor().is_valid() # simple access @@ -63,7 +66,7 @@ def test_basics(self): # exagerate the manager's overhead, but measure the buffer overhead # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! - max_num_accesses = 400 + max_num_accesses = 100 fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): for manager, man_id in ( (man_optimal, 'optimal'), From d7b486df99139ff867db01f6427408a12ff213b3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 14:26:46 +0200 Subject: [PATCH 0166/1392] Added smmap as submodule, assured the sys path makes it available --- .gitmodules | 4 ++++ gitdb/__init__.py | 16 +++++++++------- gitdb/ext/smmap | 1 + 3 files changed, 14 insertions(+), 7 deletions(-) create mode 160000 gitdb/ext/smmap diff --git a/.gitmodules b/.gitmodules index 3db4c676d..1be8ccac9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = gitdb/ext/async url = git://github.com/gitpython-developers/async.git branch = master +[submodule "smmap"] + path = gitdb/ext/smmap + url = git://github.com/Byron/smmap.git + branch = master diff --git a/gitdb/__init__.py b/gitdb/__init__.py index a551f37dc..775c969cf 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -10,13 +10,15 @@ #{ Initialization def _init_externals(): """Initialize external projects by putting them into the path""" - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', 'async')) - - try: - import async - except ImportError: - raise ImportError("'async' could not be imported, assure it is located in your PYTHONPATH") - #END verify import + for module in ('async', 'smmap'): + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) + + try: + __import__(module) + except ImportError: + raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) + #END verify import + #END handel imports #} END initialization diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap new file mode 160000 index 000000000..4466476cf --- /dev/null +++ b/gitdb/ext/smmap @@ -0,0 +1 @@ +Subproject commit 4466476cf576cf5936a11524e67345a80e2ec5a9 From d09158f9029c564c97cc33f173efd376eca873d1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 14:36:53 +0200 Subject: [PATCH 0167/1392] Made all types available in root package --- smmap/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/smmap/__init__.py b/smmap/__init__.py index 82cff638c..769858fa5 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -5,3 +5,7 @@ __homepage__ = "https://github.com/Byron/smmap" version_info = (0, 8, 0) __version__ = '.'.join(str(i) for i in version_info) + +# make everything available in root package for convenience +from mman import * +from buf import * From 9dc4a8dd154d15a243cd2d54f7d1631a913106f0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 14:42:05 +0200 Subject: [PATCH 0168/1392] Changed names to be more descriptive, hopefully. This opens op the option to implement such a manager differently, without the sliding window mechanics, which would be quite simple and not much better than a map of mmaps in the end --- smmap/buf.py | 6 +++--- smmap/mman.py | 40 ++++++++++++++++++++-------------------- smmap/test/test_buf.py | 14 +++++++------- smmap/test/test_mman.py | 12 ++++++------ smmap/test/test_util.py | 18 +++++++++--------- smmap/util.py | 16 ++++++++-------- 6 files changed, 53 insertions(+), 53 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 4fc78ea6f..94650a50c 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,11 +1,11 @@ """Module with a simple buffer implementation using the memory manager""" -from mman import MemoryCursor +from mman import SlidingCursor import sys -__all__ = ["MappedMemoryBuffer"] +__all__ = ["SlidingWindowMapBuffer"] -class MappedMemoryBuffer(object): +class SlidingWindowMapBuffer(object): """A buffer like object which allows direct byte-wise object and slicing into memory of a mapped file. The mapping is controlled by the provided cursor. diff --git a/smmap/mman.py b/smmap/mman.py index 8e50a1445..312612298 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,8 +1,8 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" from util import ( - MemoryWindow, - MappedRegion, - MappedRegionList, + MapWindow, + MapRegion, + MapRegionList, is_64_bit, ) @@ -10,16 +10,16 @@ from weakref import ref import sys -__all__ = ["MappedMemoryManager"] +__all__ = ["SlidingWindowMapManager"] #{ Utilities #}END utilities -class MemoryCursor(object): +class SlidingCursor(object): """Pointer into the mapped region of the memory manager, keeping the current window alive until it is destroyed. - Cursors should not be created manually, but are instead returned by the MappedMemoryManager""" + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager""" __slots__ = ( '_manager', # the manger keeping all file regions '_rlist', # a regions list with regions for our file @@ -29,8 +29,8 @@ class MemoryCursor(object): ) #{ Configuration - MemoryWindowCls = MemoryWindow - MappedRegionCls = MappedRegion + MapWindowCls = MapWindow + MapRegionCls = MapRegion #} END configuration def __init__(self, manager = None, regions = None): @@ -133,9 +133,9 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): #END while bisecting if existing_region is None: - left = self.MemoryWindowCls(0, 0) - mid = self.MemoryWindowCls(offset, size) - right = self.MemoryWindowCls(self.file_size(), 0) + left = self.MapWindowCls(0, 0) + mid = self.MapWindowCls(offset, size) + right = self.MapWindowCls(self.file_size(), 0) # we want to honor the max memory size, and assure we have anough # memory available @@ -166,13 +166,13 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): # possible mapping if insert_pos == 0: if len_regions: - right = self.MemoryWindowCls.from_region(a[insert_pos]) + right = self.MapWindowCls.from_region(a[insert_pos]) #END adjust right side else: if insert_pos != len_regions: - right = self.MemoryWindowCls.from_region(a[insert_pos]) + right = self.MapWindowCls.from_region(a[insert_pos]) # END adjust right window - left = self.MemoryWindowCls.from_region(a[insert_pos - 1]) + left = self.MapWindowCls.from_region(a[insert_pos - 1]) #END adjust surrounding windows mid.extend_left_to(left, window_size) @@ -189,7 +189,7 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): if man._handle_count >= man._max_handle_count: raise Exception #END assert own imposed max file handles - self._region = self.MappedRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) + self._region = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -301,7 +301,7 @@ def fd(self): #} END interface -class MappedMemoryManager(object): +class SlidingWindowMapManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. Once a certain memory limit is reached globally, or if there cannot be more open file handles @@ -315,7 +315,7 @@ class MappedMemoryManager(object): space is full.""" __slots__ = [ - '_fdict', # mapping of path -> MappedRegionList + '_fdict', # mapping of path -> MapRegionList '_window_size', # maximum size of a window '_max_memory_size', # maximum amount ofmemory we may allocate '_max_handle_count', # maximum amount of handles to keep open @@ -324,7 +324,7 @@ class MappedMemoryManager(object): ] #{ Configuration - MappedRegionListCls = MappedRegionList + MapRegionListCls = MapRegionList #} END configuration _MB_in_bytes = 1024 * 1024 @@ -411,10 +411,10 @@ def make_cursor(self, path_or_fd): prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) if regions is None: - regions = self.MappedRegionListCls(path_or_fd) + regions = self.MapRegionListCls(path_or_fd) self._fdict[path_or_fd] = regions # END obtain region for path - return MemoryCursor(self, regions) + return SlidingCursor(self, regions) def collect(self): """Collect all available free-to-collect mapped regions diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 48aeabb44..96ca4f882 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,6 +1,6 @@ from lib import TestBase, FileCreator -from smmap.mman import MappedMemoryManager +from smmap.mman import SlidingWindowMapManager from smmap.buf import * from random import randint @@ -9,8 +9,8 @@ import os -man_optimal = MappedMemoryManager() -man_worst_case = MappedMemoryManager( window_size=TestBase.k_window_test_size/100, +man_optimal = SlidingWindowMapManager() +man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, max_memory_size=TestBase.k_window_test_size/3, max_open_handles=15) @@ -21,10 +21,10 @@ def test_basics(self): # invalid paths fail upon construction c = man_optimal.make_cursor(fc.path) - self.failUnlessRaises(ValueError, MappedMemoryBuffer, type(c)()) # invalid cursor - self.failUnlessRaises(ValueError, MappedMemoryBuffer, c, fc.size) # offset too large + self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor + self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large - buf = MappedMemoryBuffer() # can create uninitailized buffers + buf = SlidingWindowMapBuffer() # can create uninitailized buffers assert buf.cursor() is None # can call end access any time @@ -71,7 +71,7 @@ def test_basics(self): for item in (fc.path, fd): for manager, man_id in ( (man_optimal, 'optimal'), (man_worst_case, 'worst case')): - buf = MappedMemoryBuffer(manager.make_cursor(item)) + buf = SlidingWindowMapBuffer(manager.make_cursor(item)) assert manager.num_file_handles() == 1 for access_mode in range(2): # single, multi num_accesses_left = max_num_accesses diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index e220f8bb2..dfe43e704 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,7 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import MemoryCursor +from smmap.mman import SlidingCursor from smmap.util import align_to_mmap from smmap.exc import RegionCollectionError @@ -16,8 +16,8 @@ class TestMMan(TestBase): def test_cursor(self): fc = FileCreator(self.k_window_test_size, "cursor_test") - man = MappedMemoryManager() - ci = MemoryCursor(man) # invalid cursor + man = SlidingWindowMapManager() + ci = SlidingCursor(man) # invalid cursor assert not ci.is_valid() assert not ci.is_associated() assert ci.size() == 0 # this is cached, so we can query it in invalid state @@ -43,10 +43,10 @@ def test_cursor(self): # destruction is fine (even multiple times) cv._destroy() - MemoryCursor(man)._destroy() + SlidingCursor(man)._destroy() def test_memory_manager(self): - man = MappedMemoryManager() + man = SlidingWindowMapManager() assert man.num_file_handles() == 0 assert man.num_open_files() == 0 assert man.window_size() > 0 @@ -82,7 +82,7 @@ def test_memman_operation(self): # small windows, a reasonable max memory. Not too many regions at once max_num_handles = 15 - man = MappedMemoryManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) + man = SlidingWindowMapManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) c = man.make_cursor(item) # still empty (more about that is tested in test_memory_manager() diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 4043cd83f..46de2ebaa 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -8,10 +8,10 @@ class TestMMan(TestBase): def test_window(self): - wl = MemoryWindow(0, 1) # left - wc = MemoryWindow(1, 1) # center - wc2 = MemoryWindow(10, 5) # another center - wr = MemoryWindow(8000, 50) # right + wl = MapWindow(0, 1) # left + wc = MapWindow(1, 1) # center + wc2 = MapWindow(10, 5) # another center + wr = MapWindow(8000, 50) # right assert wl.ofs_end() == 1 assert wc.ofs_end() == 2 @@ -56,9 +56,9 @@ def test_region(self): fc = FileCreator(self.k_window_test_size, "window_test") half_size = fc.size / 2 rofs = align_to_mmap(4200, False) - rfull = MappedRegion(fc.path, 0, fc.size) - rhalfofs = MappedRegion(fc.path, rofs, fc.size) - rhalfsize = MappedRegion(fc.path, 0, half_size) + rfull = MapRegion(fc.path, 0, fc.size) + rhalfofs = MapRegion(fc.path, rofs, fc.size) + rhalfsize = MapRegion(fc.path, 0, half_size) # offsets assert rfull.ofs_begin() == 0 and rfull.size() == fc.size @@ -88,7 +88,7 @@ def test_region(self): assert rfull.usage_count() == 1 # window constructor - w = MemoryWindow.from_region(rfull) + w = MapWindow.from_region(rfull) assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() def test_region_list(self): @@ -96,7 +96,7 @@ def test_region_list(self): fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): - ml = MappedRegionList(item) + ml = MapRegionList(item) assert ml.client_count() == 1 diff --git a/smmap/util.py b/smmap/util.py index 667777861..6ead86479 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -15,7 +15,7 @@ from sys import getrefcount __all__ = [ "align_to_mmap", "is_64_bit", - "MemoryWindow", "MappedRegion", "MappedRegionList", "ALLOCATIONGRANULARITY"] + "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities @@ -39,7 +39,7 @@ def is_64_bit(): #{ Utility Classes -class MemoryWindow(object): +class MapWindow(object): """Utility type which is used to snap windows towards each other, and to adjust their size""" __slots__ = ( 'ofs', # offset into the file in bytes @@ -51,7 +51,7 @@ def __init__(self, offset, size): self.size = size def __repr__(self): - return "MemoryWindow(%i, %i)" % (self.ofs, self.size) + return "MapWindow(%i, %i)" % (self.ofs, self.size) @classmethod def from_region(cls, region): @@ -84,7 +84,7 @@ def extend_right_to(self, window, max_size): self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) -class MappedRegion(object): +class MapRegion(object): """Defines a mapped region of memory, aligned to pagesizes :note: deallocates used region automatically on destruction""" __slots__ = [ @@ -145,7 +145,7 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): #END close file handle def __repr__(self): - return "MappedRegion<%i, %i>" % (self._b, self.size()) + return "MapRegion<%i, %i>" % (self._b, self.size()) #{ Interface @@ -201,15 +201,15 @@ def includes_ofs(self, ofs): #} END interface -class MappedRegionList(list): - """List of MappedRegion instances associating a path with a list of regions.""" +class MapRegionList(list): + """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( '_path_or_fd', # path or file descriptor which is mapped by all our regions '_file_size' # total size of the file we map ) def __new__(cls, path): - return super(MappedRegionList, cls).__new__(cls) + return super(MapRegionList, cls).__new__(cls) def __init__(self, path_or_fd): self._path_or_fd = path_or_fd From bc78951823623f9a529d0b515d85d6a0a1a1d8ac Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 15:31:07 +0200 Subject: [PATCH 0169/1392] moved code from cursor into manager, as it belongs there. This is a design error inherited from c++, but actually it makes overrides a bit harder, or lets say, less native, as the sliding mechanics where implemented in a class which is just the handle to a memory map in the end, which doesn't have to care about its allocation --- smmap/buf.py | 2 +- smmap/mman.py | 234 ++++++++++++++++++++-------------------- smmap/test/test_mman.py | 6 +- 3 files changed, 122 insertions(+), 120 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 94650a50c..741450303 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,5 +1,5 @@ """Module with a simple buffer implementation using the memory manager""" -from mman import SlidingCursor +from mman import MemoryCursor import sys diff --git a/smmap/mman.py b/smmap/mman.py index 312612298..16f878199 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -15,7 +15,7 @@ #}END utilities -class SlidingCursor(object): +class MemoryCursor(object): """Pointer into the mapped region of the memory manager, keeping the current window alive until it is destroyed. @@ -28,11 +28,6 @@ class SlidingCursor(object): '_size' # maximum size we should provide ) - #{ Configuration - MapWindowCls = MapWindow - MapRegionCls = MapRegion - #} END configuration - def __init__(self, manager = None, regions = None): self._manager = manager self._rlist = regions @@ -82,7 +77,7 @@ def assign(self, rhs): self._destroy() self._copy_from(rhs) - def use_region(self, offset, size, flags = 0, _is_recursive=False): + def use_region(self, offset, size, flags = 0): """Assure we point to a window which allows access to the given offset into the file :param offset: absolute offset in bytes into the file :param size: amount of bytes to map @@ -104,115 +99,14 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False): # END handle existing region # END check existing region + # offset too large ? + if offset >= self._rlist.file_size(): + return self + #END handle offset + if need_region: - window_size = man._window_size - - # abort on offsets beyond our mapped file's size - currently we are invalid - if offset >= self.file_size(): - return self - # END handle offset too large - - # bisect to find an existing region. The c++ implementation cannot - # do that as it uses a linked list for regions. - existing_region = None - a = self._rlist - lo = 0 - hi = len(a) - while lo < hi: - mid = (lo+hi)//2 - ofs = a[mid]._b - if ofs <= offset: - if a[mid].includes_ofs(offset): - existing_region = a[mid] - break - #END have region - lo = mid+1 - else: - hi = mid - #END handle position - #END while bisecting - - if existing_region is None: - left = self.MapWindowCls(0, 0) - mid = self.MapWindowCls(offset, size) - right = self.MapWindowCls(self.file_size(), 0) - - # we want to honor the max memory size, and assure we have anough - # memory available - # Save calls ! - if self._manager._memory_size + window_size > self._manager._max_memory_size: - man._collect_lru_region(window_size) - #END handle collection - - # we assume the list remains sorted by offset - insert_pos = 0 - len_regions = len(a) - if len_regions == 1: - if a[0]._b <= offset: - insert_pos = 1 - #END maintain sort - else: - # find insert position - insert_pos = len_regions - for i, region in enumerate(a): - if region._b > offset: - insert_pos = i - break - #END if insert position is correct - #END for each region - # END obtain insert pos - - # adjust the actual offset and size values to create the largest - # possible mapping - if insert_pos == 0: - if len_regions: - right = self.MapWindowCls.from_region(a[insert_pos]) - #END adjust right side - else: - if insert_pos != len_regions: - right = self.MapWindowCls.from_region(a[insert_pos]) - # END adjust right window - left = self.MapWindowCls.from_region(a[insert_pos - 1]) - #END adjust surrounding windows - - mid.extend_left_to(left, window_size) - mid.extend_right_to(right, window_size) - mid.align() - - # it can happen that we align beyond the end of the file - if mid.ofs_end() > right.ofs: - mid.size = right.ofs - mid.ofs - #END readjust size - - # insert new region at the right offset to keep the order - try: - if man._handle_count >= man._max_handle_count: - raise Exception - #END assert own imposed max file handles - self._region = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) - except Exception: - # apparently we are out of system resources or hit a limit - # As many more operations are likely to fail in that condition ( - # like reading a file from disk, etc) we free up as much as possible - # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway - if _is_recursive: - # we already tried this, and still have no success in obtaining - # a mapping. This is an exception, so we propagate it - raise - #END handle existing recursion - man._collect_lru_region(0) - return self.use_region(offset, size, flags, True) - #END handle exceptions - - man._handle_count += 1 - man._memory_size += self._region.size() - a.insert(insert_pos, self._region) - else: - self._region = existing_region - #END need region handling - #END handle acquire region + self._region = man._obtain_region(self._rlist, offset, size, flags, False) + #END need region handling self._region.increment_usage_count() self._ofs = offset - self._region._b @@ -301,6 +195,7 @@ def fd(self): #} END interface + class SlidingWindowMapManager(object): """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. @@ -325,6 +220,8 @@ class SlidingWindowMapManager(object): #{ Configuration MapRegionListCls = MapRegionList + MapWindowCls = MapWindow + MapRegionCls = MapRegion #} END configuration _MB_in_bytes = 1024 * 1024 @@ -398,6 +295,111 @@ def _collect_lru_region(self, size): return num_found + def _obtain_region(self, a, offset, size, flags, is_recursive): + """Utilty to create a new region - for more information on the parameters, + see MapCursor.use_region. + :param a: A regions (a)rray + :return: The newly created region""" + # bisect to find an existing region. The c++ implementation cannot + # do that as it uses a linked list for regions. + r = None + lo = 0 + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + ofs = a[mid]._b + if ofs <= offset: + if a[mid].includes_ofs(offset): + r = a[mid] + break + #END have region + lo = mid+1 + else: + hi = mid + #END handle position + #END while bisecting + + if r is None: + window_size = self._window_size + left = self.MapWindowCls(0, 0) + mid = self.MapWindowCls(offset, size) + right = self.MapWindowCls(a.file_size(), 0) + + # we want to honor the max memory size, and assure we have anough + # memory available + # Save calls ! + if self._memory_size + window_size > self._max_memory_size: + self._collect_lru_region(window_size) + #END handle collection + + # we assume the list remains sorted by offset + insert_pos = 0 + len_regions = len(a) + if len_regions == 1: + if a[0]._b <= offset: + insert_pos = 1 + #END maintain sort + else: + # find insert position + insert_pos = len_regions + for i, region in enumerate(a): + if region._b > offset: + insert_pos = i + break + #END if insert position is correct + #END for each region + # END obtain insert pos + + # adjust the actual offset and size values to create the largest + # possible mapping + if insert_pos == 0: + if len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + #END adjust right side + else: + if insert_pos != len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + # END adjust right window + left = self.MapWindowCls.from_region(a[insert_pos - 1]) + #END adjust surrounding windows + + mid.extend_left_to(left, window_size) + mid.extend_right_to(right, window_size) + mid.align() + + # it can happen that we align beyond the end of the file + if mid.ofs_end() > right.ofs: + mid.size = right.ofs - mid.ofs + #END readjust size + + # insert new region at the right offset to keep the order + try: + if self._handle_count >= self._max_handle_count: + raise Exception + #END assert own imposed max file handles + r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + #END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + a.insert(insert_pos, r) + # END create new region + return r + #{ Interface def make_cursor(self, path_or_fd): """:return: a cursor pointing to the given path or file descriptor. @@ -414,7 +416,7 @@ def make_cursor(self, path_or_fd): regions = self.MapRegionListCls(path_or_fd) self._fdict[path_or_fd] = regions # END obtain region for path - return SlidingCursor(self, regions) + return MemoryCursor(self, regions) def collect(self): """Collect all available free-to-collect mapped regions diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index dfe43e704..79c6f9892 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,7 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import SlidingCursor +from smmap.mman import MemoryCursor from smmap.util import align_to_mmap from smmap.exc import RegionCollectionError @@ -17,7 +17,7 @@ def test_cursor(self): fc = FileCreator(self.k_window_test_size, "cursor_test") man = SlidingWindowMapManager() - ci = SlidingCursor(man) # invalid cursor + ci = MemoryCursor(man) # invalid cursor assert not ci.is_valid() assert not ci.is_associated() assert ci.size() == 0 # this is cached, so we can query it in invalid state @@ -43,7 +43,7 @@ def test_cursor(self): # destruction is fine (even multiple times) cv._destroy() - SlidingCursor(man)._destroy() + MemoryCursor(man)._destroy() def test_memory_manager(self): man = SlidingWindowMapManager() From 03dd5ae25fe99b9b91212057ef57a09e40f23265 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 16:19:10 +0200 Subject: [PATCH 0170/1392] Changed design of memory managers to support different implementations. Currently there is a non-implemented static version, as well as the previous sliding window version. --- smmap/buf.py | 6 +- smmap/mman.py | 222 +++++++++++++++++++++++++++++--------------------- smmap/util.py | 4 +- 3 files changed, 134 insertions(+), 98 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 741450303..772a268e6 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -10,7 +10,11 @@ class SlidingWindowMapBuffer(object): memory of a mapped file. The mapping is controlled by the provided cursor. The buffer is relative, that is if you map an offset, index 0 will map to the - first byte at the offset you used during initialization or begin_access""" + first byte at the offset you used during initialization or begin_access + + :note: Although this type effectively hides the fact that there are mapped windows + underneath, it can unfortunately not be used in any non-pure python method which + needs a buffer or string""" __slots__ = ( '_c', # our cursor '_size', # our supposed size diff --git a/smmap/mman.py b/smmap/mman.py index 16f878199..d799c77af 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -10,7 +10,7 @@ from weakref import ref import sys -__all__ = ["SlidingWindowMapManager"] +__all__ = ["StaticWindowMapManager", "SlidingWindowMapManager"] #{ Utilities #}END utilities @@ -125,11 +125,11 @@ def unuse_region(self): def buffer(self): """Return a buffer object which allows access to our memory region from our offset - to the window size. Please note that it might be smaller than you requested + to the window size. Please note that it might be smaller than you requested when calling use_region() :note: You can only obtain a buffer if this instance is_valid() ! :note: buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" - return buffer(self._region.buffer(), self._ofs, self._size) + return buffer(self._region.map(), self._ofs, self._size) def is_valid(self): """:return: True if we have a valid and usable region""" @@ -195,19 +195,17 @@ def fd(self): #} END interface +class StaticWindowMapManager(object): + """Provides a manager which will produce single size cursors that are allowed + to always map the whole file. -class SlidingWindowMapManager(object): - """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily - obtain additional regions assuring there is no overlap. - Once a certain memory limit is reached globally, or if there cannot be more open file handles - which result from each mmap call, the least recently used, and currently unused mapped regions - are unloaded automatically. + Clients must be written to specifically know that they are accessing their data + through a StaticWindowMapManager, as they otherwise have to deal with their window size. - :note: currently not thread-safe ! - :note: in the current implementation, we will automatically unload windows if we either cannot - create more memory maps (as the open file handles limit is hit) or if we have allocated more than - a safe amount of memory already, which would possibly cause memory allocations to fail as our address - space is full.""" + These clients would have to use a SlidingWindowMapBuffer to hide this fact. + + This type will always use a maximum window size, and optimize certain methods to + acomodate this fact""" __slots__ = [ '_fdict', # mapping of path -> MapRegionList @@ -222,11 +220,12 @@ class SlidingWindowMapManager(object): MapRegionListCls = MapRegionList MapWindowCls = MapWindow MapRegionCls = MapRegion + MemoryCursorCls = MemoryCursor #} END configuration _MB_in_bytes = 1024 * 1024 - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): + def __init__(self, window_size = sys.maxint, max_memory_size = 0, max_open_handles = sys.maxint): """initialize the manager with the given parameters. :param window_size: if 0, a default window size will be chosen depending on the operating system's architechture. It will internally be quantified to a multiple of the page size @@ -258,6 +257,8 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. self._max_memory_size = coeff * self._MB_in_bytes #END handle max memory size + #{ Internal Methods + def _collect_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full @@ -265,6 +266,117 @@ def _collect_lru_region(self, size): :raise RegionCollectionError: :return: Amount of freed regions :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + raise NotImplementedError() + + def _obtain_region(self, a, offset, size, flags, is_recursive): + """Utilty to create a new region - for more information on the parameters, + see MapCursor.use_region. + :param a: A regions (a)rray + :return: The newly created region""" + raise NotImplementedError() + + #}END internal methods + + #{ Interface + def make_cursor(self, path_or_fd): + """:return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory + :note: if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open. + :note: Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" + regions = self._fdict.get(path_or_fd) + if regions is None: + regions = self.MapRegionListCls(path_or_fd) + self._fdict[path_or_fd] = regions + # END obtain region for path + return self.MemoryCursorCls(self, regions) + + def collect(self): + """Collect all available free-to-collect mapped regions + :return: Amount of freed handles""" + return self._collect_lru_region(0) + + def num_file_handles(self): + """:return: amount of file handles in use. Each mapped region uses one file handle""" + return self._handle_count + + def num_open_files(self): + """Amount of opened files in the system""" + return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) + + def window_size(self): + """:return: size of each window when allocating new regions""" + return self._window_size + + def mapped_memory_size(self): + """:return: amount of bytes currently mapped in total""" + return self._memory_size + + def max_file_handles(self): + """:return: maximium amount of handles we may have opened""" + return self._max_handle_count + + def max_mapped_memory_size(self): + """:return: maximum amount of memory we may allocate""" + return self._max_memory_size + + #} END interface + + #{ Special Purpose Interface + + def force_map_handle_removal_win(self, base_path): + """ONLY AVAILABLE ON WINDOWS + On windows removing files is not allowed if anybody still has it opened. + If this process is ourselves, and if the whole process uses this memory + manager (as far as the parent framework is concerned) we can enforce + closing all memory maps whose path matches the given base path to + allow the respective operation after all. + The respective system must NOT access the closed memory regions anymore ! + This really may only be used if you know that the items which keep + the cursors alive will not be using it anymore. They need to be recreated ! + :return: Amount of closed handles + :note: does nothing on non-windows platforms""" + if sys.platform != 'win32': + return + #END early bailout + + num_closed = 0 + for path, rlist in self._fdict.iteritems(): + if path.startswith(base_path): + for region in rlist: + region._mf.close() + num_closed += 1 + #END path matches + #END for each path + return num_closed + #} END special purpose interface + + + +class SlidingWindowMapManager(StaticWindowMapManager): + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily + obtain additional regions assuring there is no overlap. + Once a certain memory limit is reached globally, or if there cannot be more open file handles + which result from each mmap call, the least recently used, and currently unused mapped regions + are unloaded automatically. + + :note: currently not thread-safe ! + :note: in the current implementation, we will automatically unload windows if we either cannot + create more memory maps (as the open file handles limit is hit) or if we have allocated more than + a safe amount of memory already, which would possibly cause memory allocations to fail as our address + space is full.""" + + __slots__ = tuple() + + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): + """Adjusts the default window size to 0""" + super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) + + def _collect_lru_region(self, size): num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): lru_region = None @@ -296,10 +408,6 @@ def _collect_lru_region(self, size): return num_found def _obtain_region(self, a, offset, size, flags, is_recursive): - """Utilty to create a new region - for more information on the parameters, - see MapCursor.use_region. - :param a: A regions (a)rray - :return: The newly created region""" # bisect to find an existing region. The c++ implementation cannot # do that as it uses a linked list for regions. r = None @@ -400,80 +508,4 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # END create new region return r - #{ Interface - def make_cursor(self, path_or_fd): - """:return: a cursor pointing to the given path or file descriptor. - It can be used to map new regions of the file into memory - :note: if a file descriptor is given, it is assumed to be open and valid, - but may be closed afterwards. To refer to the same file, you may reuse - your existing file descriptor, but keep in mind that new windows can only - be mapped as long as it stays valid. This is why the using actual file paths - are preferred unless you plan to keep the file descriptor open. - :note: Using file descriptors directly is faster once new windows are mapped as it - prevents the file to be opened again just for the purpose of mapping it.""" - regions = self._fdict.get(path_or_fd) - if regions is None: - regions = self.MapRegionListCls(path_or_fd) - self._fdict[path_or_fd] = regions - # END obtain region for path - return MemoryCursor(self, regions) - - def collect(self): - """Collect all available free-to-collect mapped regions - :return: Amount of freed handles""" - return self._collect_lru_region(0) - - def num_file_handles(self): - """:return: amount of file handles in use. Each mapped region uses one file handle""" - return self._handle_count - def num_open_files(self): - """Amount of opened files in the system""" - return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) - - def window_size(self): - """:return: size of each window when allocating new regions""" - return self._window_size - - def mapped_memory_size(self): - """:return: amount of bytes currently mapped in total""" - return self._memory_size - - def max_file_handles(self): - """:return: maximium amount of handles we may have opened""" - return self._max_handle_count - - def max_mapped_memory_size(self): - """:return: maximum amount of memory we may allocate""" - return self._max_memory_size - - #} END interface - - #{ Special Purpose Interface - - def force_map_handle_removal_win(self, base_path): - """ONLY AVAILABLE ON WINDOWS - On windows removing files is not allowed if anybody still has it opened. - If this process is ourselves, and if the whole process uses this memory - manager (as far as the parent framework is concerned) we can enforce - closing all memory maps whose path matches the given base path to - allow the respective operation after all. - The respective system must NOT access the closed memory regions anymore ! - This really may only be used if you know that the items which keep - the cursors alive will not be using it anymore. They need to be recreated ! - :return: Amount of closed handles - :note: does nothing on non-windows platforms""" - if sys.platform != 'win32': - return - #END early bailout - - num_closed = 0 - for path, rlist in self._fdict.iteritems(): - if path.startswith(base_path): - for region in rlist: - region._mf.close() - num_closed += 1 - #END path matches - #END for each path - return num_closed - #} END special purpose interface diff --git a/smmap/util.py b/smmap/util.py index 6ead86479..21e285559 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -149,8 +149,8 @@ def __repr__(self): #{ Interface - def buffer(self): - """:return: a sliceable buffer which can be used to access the mapped memory""" + def map(self): + """:return: a memory map containing the memory""" return self._mf def ofs_begin(self): From 631b9ea2edfb005b1d48d55b35370f82ac2de196 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 17:47:48 +0200 Subject: [PATCH 0171/1392] Implemented static memory manager, for now without test --- smmap/buf.py | 2 +- smmap/mman.py | 98 ++++++++++++++++++++++++++++++++++++----- smmap/test/test_mman.py | 14 ++++-- 3 files changed, 99 insertions(+), 15 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 772a268e6..c4d252251 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,5 +1,5 @@ """Module with a simple buffer implementation using the memory manager""" -from mman import MemoryCursor +from mman import WindowCursor import sys diff --git a/smmap/mman.py b/smmap/mman.py index d799c77af..ba3a63f55 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -9,17 +9,23 @@ from exc import RegionCollectionError from weakref import ref import sys +from sys import getrefcount __all__ = ["StaticWindowMapManager", "SlidingWindowMapManager"] #{ Utilities #}END utilities -class MemoryCursor(object): - """Pointer into the mapped region of the memory manager, keeping the current window - alive until it is destroyed. + + +class WindowCursor(object): + """Pointer into the mapped region of the memory manager, keeping the map + alive until it is destroyed and no other client uses it. - Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager""" + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager + :note: The current implementation is suited for static and sliding window managers, but it also means + that it must be suited for the somewhat quite different sliding manager. It could be improved, but + I see no real need to do so.""" __slots__ = ( '_manager', # the manger keeping all file regions '_rlist', # a regions list with regions for our file @@ -130,7 +136,14 @@ def buffer(self): :note: buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.map(), self._ofs, self._size) - + + def map(self): + """ + :return: the underlying raw memory map. Please not that the offset and size is likely to be different + to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole + file in case of StaticWindowMapManager""" + return self._region.map() + def is_valid(self): """:return: True if we have a valid and usable region""" return self._region is not None @@ -188,7 +201,7 @@ def fd(self): :note: it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), basestring): - return ValueError("File descriptor queried although mapping was generated from path") + raise ValueError("File descriptor queried although mapping was generated from path") #END handle type return self._rlist.path_or_fd() @@ -208,7 +221,7 @@ class StaticWindowMapManager(object): acomodate this fact""" __slots__ = [ - '_fdict', # mapping of path -> MapRegionList + '_fdict', # mapping of path -> StorageHelper (of some kind '_window_size', # maximum size of a window '_max_memory_size', # maximum amount ofmemory we may allocate '_max_handle_count', # maximum amount of handles to keep open @@ -220,7 +233,7 @@ class StaticWindowMapManager(object): MapRegionListCls = MapRegionList MapWindowCls = MapWindow MapRegionCls = MapRegion - MemoryCursorCls = MemoryCursor + WindowCursorCls = WindowCursor #} END configuration _MB_in_bytes = 1024 * 1024 @@ -266,14 +279,77 @@ def _collect_lru_region(self, size): :raise RegionCollectionError: :return: Amount of freed regions :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" - raise NotImplementedError() + num_found = 0 + while (size == 0) or (self._memory_size + size > self._max_memory_size): + for k, regions in self._fdict.iteritems(): + found_lonely_region = False + for region in regions: + # check client count - consider that we keep one reference ourselves ! + if (region.client_count()-2 == 0 and + (lru_region is None or region._uc < lru_region._uc)): + # remove whole list + found_lonely_region = True + num_found += 1 + self._memory_size -= region.size() + self._handle_count -= 1 + self._fdict.pop(k) + + break + # END update lru_region + #END for each region + if found_lonely_region: + continue + # END skip iteration and restart + #END for each regions list + + # still here ? + if num_found == 0 and size != 0: + raise RegionCollectionError("Didn't find any region to free") + #END raise if necessary + #END while there is more memory to free + + return num_found + def _obtain_region(self, a, offset, size, flags, is_recursive): """Utilty to create a new region - for more information on the parameters, see MapCursor.use_region. :param a: A regions (a)rray :return: The newly created region""" - raise NotImplementedError() + if self._memory_size + window_size > self._max_memory_size: + self._collect_lru_region(window_size) + #END handle collection + + r = None + if a: + assert len(a) == 1 + r = a[0] + else: + try: + r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxint, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + #END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + # END handle array + + assert a.includes_ofs(offset) + assert a.includes_ofs(offset + size-1) + return r #}END internal methods @@ -293,7 +369,7 @@ def make_cursor(self, path_or_fd): regions = self.MapRegionListCls(path_or_fd) self._fdict[path_or_fd] = regions # END obtain region for path - return self.MemoryCursorCls(self, regions) + return self.WindowCursorCls(self, regions) def collect(self): """Collect all available free-to-collect mapped regions diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 79c6f9892..e8266066c 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,7 +1,7 @@ from lib import TestBase, FileCreator from smmap.mman import * -from smmap.mman import MemoryCursor +from smmap.mman import WindowCursor from smmap.util import align_to_mmap from smmap.exc import RegionCollectionError @@ -17,7 +17,7 @@ def test_cursor(self): fc = FileCreator(self.k_window_test_size, "cursor_test") man = SlidingWindowMapManager() - ci = MemoryCursor(man) # invalid cursor + ci = WindowCursor(man) # invalid cursor assert not ci.is_valid() assert not ci.is_associated() assert ci.size() == 0 # this is cached, so we can query it in invalid state @@ -43,7 +43,7 @@ def test_cursor(self): # destruction is fine (even multiple times) cv._destroy() - MemoryCursor(man)._destroy() + WindowCursor(man)._destroy() def test_memory_manager(self): man = SlidingWindowMapManager() @@ -65,11 +65,19 @@ def test_memory_manager(self): fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): c = man.make_cursor(item) + assert c.path_or_fd() is item assert c.use_region(10, 10).is_valid() assert c.ofs_begin() == 10 assert c.size() == 10 assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + + if isinstance(item, int): + self.failUnlessRaises(ValueError, c.path) + else: + self.failUnlessRaises(ValueError, c.fd) + #END handle value error #END for each input + os.close(fd) def test_memman_operation(self): From e010084b0f40d7762f029a7ac4109c9bb99818a6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 18:28:37 +0200 Subject: [PATCH 0172/1392] test are running, once again, but not yet complete regarding the static manager --- smmap/mman.py | 92 ++++++++++++++--------------------------- smmap/test/test_mman.py | 71 +++++++++++++++++-------------- 2 files changed, 70 insertions(+), 93 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index ba3a63f55..bdefe2e81 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -95,7 +95,8 @@ def use_region(self, offset, size, flags = 0): either the file has reached its end, or the map was created between two existing regions""" need_region = True man = self._manager - size = min(size, man.window_size()) # clamp size to window size + fsize = self._rlist.file_size() + size = min(size, man.window_size() or fsize) # clamp size to window size if self._region is not None: if self._region.includes_ofs(offset): @@ -106,7 +107,7 @@ def use_region(self, offset, size, flags = 0): # END check existing region # offset too large ? - if offset >= self._rlist.file_size(): + if offset >= fsize: return self #END handle offset @@ -238,10 +239,11 @@ class StaticWindowMapManager(object): _MB_in_bytes = 1024 * 1024 - def __init__(self, window_size = sys.maxint, max_memory_size = 0, max_open_handles = sys.maxint): + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): """initialize the manager with the given parameters. - :param window_size: if 0, a default window size will be chosen depending on + :param window_size: if -1, a default window size will be chosen depending on the operating system's architechture. It will internally be quantified to a multiple of the page size + If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default iwll be set dependning on the system's architecture. :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. @@ -254,7 +256,7 @@ def __init__(self, window_size = sys.maxint, max_memory_size = 0, max_open_handl self._memory_size = 0 self._handle_count = 0 - if window_size == 0: + if window_size < 0: coeff = 32 if is_64_bit(): coeff = 1024 @@ -281,43 +283,40 @@ def _collect_lru_region(self, size): :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): - for k, regions in self._fdict.iteritems(): - found_lonely_region = False + lru_region = None + lru_list = None + for regions in self._fdict.itervalues(): for region in regions: # check client count - consider that we keep one reference ourselves ! if (region.client_count()-2 == 0 and (lru_region is None or region._uc < lru_region._uc)): - # remove whole list - found_lonely_region = True - num_found += 1 - self._memory_size -= region.size() - self._handle_count -= 1 - self._fdict.pop(k) - - break + lru_region = region + lru_list = regions # END update lru_region #END for each region - if found_lonely_region: - continue - # END skip iteration and restart #END for each regions list - # still here ? - if num_found == 0 and size != 0: - raise RegionCollectionError("Didn't find any region to free") - #END raise if necessary + if lru_region is None: + if num_found == 0 and size != 0: + raise RegionCollectionError("Didn't find any region to free") + #END raise if necessary + break + #END handle region not found + + num_found += 1 + del(lru_list[lru_list.index(lru_region)]) + self._memory_size -= lru_region.size() + self._handle_count -= 1 #END while there is more memory to free - return num_found - def _obtain_region(self, a, offset, size, flags, is_recursive): """Utilty to create a new region - for more information on the parameters, see MapCursor.use_region. :param a: A regions (a)rray :return: The newly created region""" - if self._memory_size + window_size > self._max_memory_size: - self._collect_lru_region(window_size) + if self._memory_size + size > self._max_memory_size: + self._collect_lru_region(size) #END handle collection r = None @@ -347,8 +346,8 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): self._memory_size += r.size() # END handle array - assert a.includes_ofs(offset) - assert a.includes_ofs(offset + size-1) + assert r.includes_ofs(offset) + assert r.includes_ofs(offset + size-1) return r #}END internal methods @@ -362,6 +361,8 @@ def make_cursor(self, path_or_fd): your existing file descriptor, but keep in mind that new windows can only be mapped as long as it stays valid. This is why the using actual file paths are preferred unless you plan to keep the file descriptor open. + :note: file descriptors are problematic as they are not necessarily unique, as two + different files opened and closed in succession might have the same file descriptor id. :note: Using file descriptors directly is faster once new windows are mapped as it prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) @@ -448,41 +449,10 @@ class SlidingWindowMapManager(StaticWindowMapManager): __slots__ = tuple() - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): - """Adjusts the default window size to 0""" + def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxint): + """Adjusts the default window size to -1""" super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) - def _collect_lru_region(self, size): - num_found = 0 - while (size == 0) or (self._memory_size + size > self._max_memory_size): - lru_region = None - lru_list = None - for regions in self._fdict.itervalues(): - for region in regions: - # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-2 == 0 and - (lru_region is None or region._uc < lru_region._uc)): - lru_region = region - lru_list = regions - # END update lru_region - #END for each region - #END for each regions list - - if lru_region is None: - if num_found == 0 and size != 0: - raise RegionCollectionError("Didn't find any region to free") - #END raise if necessary - break - #END handle region not found - - num_found += 1 - del(lru_list[lru_list.index(lru_region)]) - self._memory_size -= lru_region.size() - self._handle_count -= 1 - #END while there is more memory to free - - return num_found - def _obtain_region(self, a, offset, size, flags, is_recursive): # bisect to find an existing region. The c++ implementation cannot # do that as it uses a linked list for regions. diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index e8266066c..97866bbf9 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -46,40 +46,47 @@ def test_cursor(self): WindowCursor(man)._destroy() def test_memory_manager(self): - man = SlidingWindowMapManager() - assert man.num_file_handles() == 0 - assert man.num_open_files() == 0 - assert man.window_size() > 0 - assert man.mapped_memory_size() == 0 - assert man.max_mapped_memory_size() > 0 - - # collection doesn't raise in 'any' mode - man._collect_lru_region(0) - # doesn't raise if we are within the limit - man._collect_lru_region(10) - # raises outside of limit - self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) + slide_man = SlidingWindowMapManager() + static_man = StaticWindowMapManager() - # use a region, verify most basic functionality - fc = FileCreator(self.k_window_test_size, "manager_test") - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - c = man.make_cursor(item) - assert c.path_or_fd() is item - assert c.use_region(10, 10).is_valid() - assert c.ofs_begin() == 10 - assert c.size() == 10 - assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + for man in (static_man, slide_man): + assert man.num_file_handles() == 0 + assert man.num_open_files() == 0 + winsize_cmp_val = 0 + if isinstance(man, StaticWindowMapManager): + winsize_cmp_val = -1 + #END handle window size + assert man.window_size() > winsize_cmp_val + assert man.mapped_memory_size() == 0 + assert man.max_mapped_memory_size() > 0 + + # collection doesn't raise in 'any' mode + man._collect_lru_region(0) + # doesn't raise if we are within the limit + man._collect_lru_region(10) + # raises outside of limit + self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) + + # use a region, verify most basic functionality + fc = FileCreator(self.k_window_test_size, "manager_test") + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + c = man.make_cursor(item) + assert c.path_or_fd() is item + assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + + if isinstance(item, int): + self.failUnlessRaises(ValueError, c.path) + else: + self.failUnlessRaises(ValueError, c.fd) + #END handle value error + #END for each input + os.close(fd) + # END for each manager type - if isinstance(item, int): - self.failUnlessRaises(ValueError, c.path) - else: - self.failUnlessRaises(ValueError, c.fd) - #END handle value error - #END for each input - - os.close(fd) - def test_memman_operation(self): # test more access, force it to actually unmap regions fc = FileCreator(self.k_window_test_size, "manager_operation_test") From 101a4d39108c4b4731760456523fd0f0d565b6f2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 19:50:43 +0200 Subject: [PATCH 0173/1392] Finally the mem manager and buffer tests run with the static one too --- smmap/mman.py | 18 ++-- smmap/test/test_buf.py | 6 +- smmap/test/test_mman.py | 216 ++++++++++++++++++++++------------------ 3 files changed, 131 insertions(+), 109 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index bdefe2e81..f1ce95d03 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -4,9 +4,9 @@ MapRegion, MapRegionList, is_64_bit, + align_to_mmap ) -from exc import RegionCollectionError from weakref import ref import sys from sys import getrefcount @@ -83,10 +83,10 @@ def assign(self, rhs): self._destroy() self._copy_from(rhs) - def use_region(self, offset, size, flags = 0): + def use_region(self, offset, size = 0, flags = 0): """Assure we point to a window which allows access to the given offset into the file :param offset: absolute offset in bytes into the file - :param size: amount of bytes to map + :param size: amount of bytes to map. If 0, all available bytes will be mapped :param flags: additional flags to be given to os.open in case a file handle is initially opened for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. @@ -96,7 +96,7 @@ def use_region(self, offset, size, flags = 0): need_region = True man = self._manager fsize = self._rlist.file_size() - size = min(size, man.window_size() or fsize) # clamp size to window size + size = min(size or fsize, man.window_size() or fsize) # clamp size to window size if self._region is not None: if self._region.includes_ofs(offset): @@ -246,6 +246,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default iwll be set dependning on the system's architecture. + It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, the manager will free as many handles as posisble""" @@ -278,8 +279,9 @@ def _collect_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region - :raise RegionCollectionError: :return: Amount of freed regions + :note: We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. + If the system runs out of memory, it will tell. :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): @@ -297,9 +299,6 @@ def _collect_lru_region(self, size): #END for each regions list if lru_region is None: - if num_found == 0 and size != 0: - raise RegionCollectionError("Didn't find any region to free") - #END raise if necessary break #END handle region not found @@ -344,10 +343,11 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): self._handle_count += 1 self._memory_size += r.size() + a.append(r) # END handle array assert r.includes_ofs(offset) - assert r.includes_ofs(offset + size-1) + #assert r.includes_ofs(offset+size-1) return r #}END internal methods diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 96ca4f882..d8b7fbcab 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,6 +1,6 @@ from lib import TestBase, FileCreator -from smmap.mman import SlidingWindowMapManager +from smmap.mman import SlidingWindowMapManager, StaticWindowMapManager from smmap.buf import * from random import randint @@ -13,6 +13,7 @@ man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, max_memory_size=TestBase.k_window_test_size/3, max_open_handles=15) +static_man = StaticWindowMapManager() class TestBuf(TestBase): @@ -70,7 +71,8 @@ def test_basics(self): fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): for manager, man_id in ( (man_optimal, 'optimal'), - (man_worst_case, 'worst case')): + (man_worst_case, 'worst case'), + (static_man, 'static optimial')): buf = SlidingWindowMapBuffer(manager.make_cursor(item)) assert manager.num_file_handles() == 1 for access_mode in range(2): # single, multi diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 97866bbf9..27be686ad 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -64,8 +64,9 @@ def test_memory_manager(self): man._collect_lru_region(0) # doesn't raise if we are within the limit man._collect_lru_region(10) - # raises outside of limit - self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint) + + # doesn't fail if we overallocate + assert man._collect_lru_region(sys.maxint) == 0 # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") @@ -92,103 +93,122 @@ def test_memman_operation(self): fc = FileCreator(self.k_window_test_size, "manager_operation_test") data = open(fc.path, 'rb').read() fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - assert len(data) == fc.size - - # small windows, a reasonable max memory. Not too many regions at once - max_num_handles = 15 - man = SlidingWindowMapManager(window_size=fc.size / 100, max_memory_size=fc.size / 3, max_open_handles=max_num_handles) - c = man.make_cursor(item) - - # still empty (more about that is tested in test_memory_manager() - assert man.num_open_files() == 0 - assert man.mapped_memory_size() == 0 - - base_offset = 5000 - size = man.window_size() / 2 - assert c.use_region(base_offset, size).is_valid() - rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us - - assert man.num_open_files() == 1 - assert man.num_file_handles() == 1 - assert man.mapped_memory_size() == rr().size() - assert c.size() == size - assert c.ofs_begin() == base_offset - assert rr().ofs_begin() == 0 # it was aligned and expanded - assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) - - assert c.buffer()[:] == data[base_offset:base_offset+size] - - # obtain second window, which spans the first part of the file - it is a still the same window - assert c.use_region(0, size-10).is_valid() - assert c.region_ref()() == rr() - assert man.num_file_handles() == 1 - assert c.size() == size-10 - assert c.ofs_begin() == 0 - assert c.buffer()[:] == data[:size-10] - - # map some part at the end, our requested size cannot be kept - overshoot = 4000 - base_offset = fc.size - size + overshoot - assert c.use_region(base_offset, size).is_valid() - assert man.num_file_handles() == 2 - assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager - rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset+size] - - # unising a region makes the cursor invalid - c.unuse_region() - assert not c.is_valid() - # but doesn't change anything regarding the handle count - we cache it and only - # remove mapped regions if we have to - assert man.num_file_handles() == 2 - - # iterate through the windows, verify data contents - # this will trigger map collection after a while - max_random_accesses = 5000 - num_random_accesses = max_random_accesses - memory_read = 0 - st = time() - - # cache everything to get some more performance - includes_ofs = c.includes_ofs - max_mapped_memory_size = man.max_mapped_memory_size() - max_file_handles = man.max_file_handles() - mapped_memory_size = man.mapped_memory_size - num_file_handles = man.num_file_handles - while num_random_accesses: - num_random_accesses -= 1 - base_offset = randint(0, fc.size - 1) + max_num_handles = 15 + #small_size = + for mtype, args in ( (StaticWindowMapManager, (0, fc.size / 3, max_num_handles)), + (SlidingWindowMapManager, (fc.size / 100, fc.size / 3, max_num_handles)),): + for item in (fc.path, fd): + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) + c = man.make_cursor(item) - # precondition - assert max_mapped_memory_size >= mapped_memory_size() - assert max_file_handles >= num_file_handles() + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + # window size is 0 for static managers, hence size will be 0. We take that into consideration + size = man.window_size() / 2 assert c.use_region(base_offset, size).is_valid() - csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset+csize] - memory_read += csize + rr = c.region_ref() + assert rr().client_count() == 2 # the manager and the cursor and us - assert includes_ofs(base_offset) - assert includes_ofs(base_offset+csize-1) - assert not includes_ofs(base_offset+csize) - # END while we should do an access - elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows - mb = float(1000 * 1000) - sys.stderr.write("Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) - - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - - # collection - it should be able to collect all - assert man.num_file_handles() - assert man.collect() - assert man.num_file_handles() == 0 - #END for each item + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr().size() + + #assert c.size() == size # the cursor may overallocate in its static version + assert c.ofs_begin() == base_offset + assert rr().ofs_begin() == 0 # it was aligned and expanded + if man.window_size(): + assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) + else: + assert rr().size() == fc.size + #END ignore static managers which dont use windows and are aligned to file boundaries + + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + + # obtain second window, which spans the first part of the file - it is a still the same window + nsize = (size or fc.size) - 10 + assert c.use_region(0, nsize).is_valid() + assert c.region_ref()() == rr() + assert man.num_file_handles() == 1 + assert c.size() == nsize + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:nsize] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - (size or c.size()) + overshoot + assert c.use_region(base_offset, size).is_valid() + if man.window_size(): + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager + else: + assert c.size() < fc.size + #END ignore static managers which only have one handle per file + rr = c.region_ref() + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + if man.window_size(): + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + #END ignore this for static managers + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 5000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles + while num_random_accesses: + num_random_accesses -= 1 + base_offset = randint(0, fc.size - 1) + + # precondition + if man.window_size(): + assert max_mapped_memory_size >= mapped_memory_size() + #END statics will overshoot, which is fine + assert max_file_handles >= num_file_handles() + assert c.use_region(base_offset, (size or c.size())).is_valid() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset+csize] + memory_read += csize + + assert includes_ofs(base_offset) + assert includes_ofs(base_offset+csize-1) + assert not includes_ofs(base_offset+csize) + # END while we should do an access + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows + mb = float(1000 * 1000) + sys.stderr.write("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 + #END for each item + # END for each manager type os.close(fd) From 82c97ea8f416bd99f501608b2aee8b7c4c5e78f5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 20:12:40 +0200 Subject: [PATCH 0174/1392] Some more adjustments to make it work in all python versions --- smmap/mman.py | 2 +- smmap/util.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index f1ce95d03..fc6848413 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -136,7 +136,7 @@ def buffer(self): :note: You can only obtain a buffer if this instance is_valid() ! :note: buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" - return buffer(self._region.map(), self._ofs, self._size) + return buffer(self._region.buffer(), self._ofs, self._size) def map(self): """ diff --git a/smmap/util.py b/smmap/util.py index 21e285559..80ba3c5b9 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -136,7 +136,7 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): self._size = len(self._mf) if self._need_compat_layer: - self._mfb = buffer(self._mf, ofs, size) + self._mfb = buffer(self._mf, ofs, self._size) #END handle buffer wrapping finally: if isinstance(path_or_fd, basestring): @@ -148,7 +148,11 @@ def __repr__(self): return "MapRegion<%i, %i>" % (self._b, self.size()) #{ Interface - + + def buffer(self): + """:return: a buffer containing the memory""" + return self._mf + def map(self): """:return: a memory map containing the memory""" return self._mf From a33e8d55d4d77d842edea94a78d801b23bb90294 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 21:59:22 +0200 Subject: [PATCH 0175/1392] Switched git db to the non-sliding version of the memory manager which is a good tradeoff between performance loss and resource handling --- gitdb/ext/smmap | 2 +- gitdb/pack.py | 76 +++++++++++++++++++++++-------------------------- gitdb/util.py | 13 +++++++++ 3 files changed, 50 insertions(+), 41 deletions(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 4466476cf..4cabaca18 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 4466476cf576cf5936a11524e67345a80e2ec5a9 +Subproject commit 4cabaca18cd30399288fc55863de412446ea51e7 diff --git a/gitdb/pack.py b/gitdb/pack.py index 7ae9786e6..0679a6ecf 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -10,10 +10,10 @@ ) from util import ( zlib, + mman, LazyMixin, unpack_from, bin_to_hex, - file_contents_ro_filepath, ) from fun import ( @@ -247,7 +247,7 @@ class PackIndexFile(LazyMixin): # Dont use slots as we dynamically bind functions for each version, need a dict for this # The slots you see here are just to keep track of our instance variables - # __slots__ = ('_indexpath', '_fanout_table', '_data', '_version', + # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') # used in v2 indices @@ -261,22 +261,23 @@ def __init__(self, indexpath): def _set_cache_(self, attr): if attr == "_packfile_checksum": - self._packfile_checksum = self._data[-40:-20] + self._packfile_checksum = self._cursor.map()[-40:-20] elif attr == "_packfile_checksum": - self._packfile_checksum = self._data[-20:] - elif attr == "_data": + self._packfile_checksum = self._cursor.map()[-20:] + elif attr == "_cursor": # Note: We don't lock the file when reading as we cannot be sure # that we can actually write to the location - it could be a read-only # alternate for instance - self._data = file_contents_ro_filepath(self._indexpath) + self._cursor = mman.make_cursor(self._indexpath).use_region() else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties # CHECK VERSION - self._version = (self._data[:4] == self.index_v2_signature and 2) or 1 + mmap = self._cursor.map() + self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 if self._version == 2: - version_id = unpack_from(">L", self._data, 4)[0] + version_id = unpack_from(">L", mmap, 4)[0] assert version_id == self._version, "Unsupported index version: %i" % version_id # END assert version @@ -297,16 +298,16 @@ def _set_cache_(self, attr): def _entry_v1(self, i): """:return: tuple(offset, binsha, 0)""" - return unpack_from(">L20s", self._data, 1024 + i*24) + (0, ) + return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) def _offset_v1(self, i): """see ``_offset_v2``""" - return unpack_from(">L", self._data, 1024 + i*24)[0] + return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] def _sha_v1(self, i): """see ``_sha_v2``""" base = 1024 + (i*24)+4 - return self._data[base:base+20] + return self._cursor.map()[base:base+20] def _crc_v1(self, i): """unsupported""" @@ -322,13 +323,13 @@ def _entry_v2(self, i): def _offset_v2(self, i): """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only be returned if the pack is larger than 4 GiB, or 2^32""" - offset = unpack_from(">L", self._data, self._pack_offset + i * 4)[0] + offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] # if the high-bit is set, this indicates that we have to lookup the offset # in the 64 bit region of the file. The current offset ( lower 31 bits ) # are the index into it if offset & 0x80000000: - offset = unpack_from(">Q", self._data, self._pack_64_offset + (offset & ~0x80000000) * 8)[0] + offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] # END handle 64 bit offset return offset @@ -336,11 +337,11 @@ def _offset_v2(self, i): def _sha_v2(self, i): """:return: sha at the given index of this file index instance""" base = self._sha_list_offset + i * 20 - return self._data[base:base+20] + return self._cursor.map()[base:base+20] def _crc_v2(self, i): """:return: 4 bytes crc for the object at index i""" - return unpack_from(">L", self._data, self._crc_list_offset + i * 4)[0] + return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] #} END access V2 @@ -358,7 +359,7 @@ def _initialize(self): def _read_fanout(self, byte_offset): """Generate a fanout table from our data""" - d = self._data + d = self._cursor.map() out = list() append = out.append for i in range(256): @@ -382,11 +383,11 @@ def path(self): def packfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of the pack file""" - return self._data[-40:-20] + return self._cursor.map()[-40:-20] def indexfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of this index file""" - return self._data[-20:] + return self._cursor.map()[-20:] def offsets(self): """:return: sequence of all offsets in the order in which they were written @@ -394,7 +395,7 @@ def offsets(self): if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears - a.fromstring(buffer(self._data, self._pack_offset, self._pack_64_offset - self._pack_offset)) + a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) # networkbyteorder to something array likes more if sys.byteorder == 'little': @@ -501,7 +502,7 @@ class PackFile(LazyMixin): for some reason - one clearly doesn't want to read 10GB at once in that case""" - __slots__ = ('_packpath', '_data', '_size', '_version') + __slots__ = ('_packpath', '_cursor', '_size', '_version') pack_signature = 0x5041434b # 'PACK' pack_version_default = 2 @@ -513,26 +514,20 @@ def __init__(self, packpath): self._packpath = packpath def _set_cache_(self, attr): - if attr == '_data': - self._data = file_contents_ro_filepath(self._packpath) - - # read the header information - type_id, self._version, self._size = unpack_from(">LLL", self._data, 0) - - # TODO: figure out whether we should better keep the lock, or maybe - # add a .keep file instead ? - else: # must be '_size' or '_version' - # read header info - we do that just with a file stream - type_id, self._version, self._size = unpack(">LLL", open(self._packpath).read(12)) - # END handle header + # we fill the whole cache, whichever attribute gets queried first + self._cursor = mman.make_cursor(self._packpath).use_region() + # read the header information + type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) + + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? if type_id != self.pack_signature: raise ParseError("Invalid pack signature: %i" % type_id) - #END assert type id def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" - data = self._data + data = self._cursor.map() content_size = len(data) - self.footer_size cur_offset = start_offset or self.first_object_offset @@ -568,11 +563,11 @@ def data(self): """ :return: read-only data of this pack. It provides random access and usually is a memory map""" - return self._data + return self._cursor.map() def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" - return self._data[-20:] + return self._cursor.map()[-20:] def path(self): """:return: path to the packfile""" @@ -591,8 +586,9 @@ def collect_streams(self, offset): If the object at offset is no delta, the size of the list is 1. :param offset: specifies the first byte of the object within this pack""" out = list() + data = self._cursor.map() while True: - ostream = pack_object_at(self._data, offset, True)[1] + ostream = pack_object_at(data, offset, True)[1] out.append(ostream) if ostream.type_id == OFS_DELTA: offset = ostream.pack_offset - ostream.delta_info @@ -614,14 +610,14 @@ def info(self, offset): :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self.first_object_offset, False)[1] + return pack_object_at(self._cursor.map(), offset or self.first_object_offset, False)[1] def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._data, offset or self.first_object_offset, True)[1] + return pack_object_at(self._cursor.map(), offset or self.first_object_offset, True)[1] def stream_iter(self, start_offset=0): """ @@ -704,7 +700,7 @@ def _object(self, sha, as_stream, index=-1): sha = self._index.sha(index) # END assure sha is present ( in output ) offset = self._index.offset(index) - type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._data, offset)) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._cursor.map(), offset)) if as_stream: if type_id not in delta_types: packstream = self._pack.stream(offset) diff --git a/gitdb/util.py b/gitdb/util.py index 4bb3c7352..4ce615585 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -23,6 +23,14 @@ # END try async zlib from async import ThreadPool +from smmap import ( + StaticWindowMapManager, + SlidingWindowMapBuffer + ) + +# initialize our global memory manager instance +# Use it to free cached (and unused) resources. +mman = StaticWindowMapManager() try: import hashlib @@ -180,6 +188,11 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): close(fd) # END assure file is closed +def sliding_ro_buffer(filepath, flags=0): + """:return: a buffer compatible object which uses our mapped memory manager internally + ready to read the whole given filepath""" + return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: From 34f01396b913220fe5b19e1f8e33f2d3f4ec2ce5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 22:05:36 +0200 Subject: [PATCH 0176/1392] Added changelog information --- doc/source/changes.rst | 5 +++++ gitdb/test/performance/test_pack.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 7b8ebecc6..999cc1309 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,11 @@ Changelog ######### +***** +0.5.3 +***** +* Added support for smmap. SmartMMap allows resources to be managed and controlled. This brings the implementation closer to the way git handles memory maps, such that unused cached memory maps will automatically be freed once a resource limit is hit. The memory limit on 32 bit systems remains though as a sliding mmap implementation is not used for performance reasons. + ***** 0.5.2 ***** diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index da952b17a..20618024d 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -15,9 +15,11 @@ from time import time import random +from nose import SkipTest + class TestPackedDBPerformance(TestBigRepoR): - def _test_pack_random_access(self): + def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # sha lookup @@ -66,6 +68,7 @@ def _test_pack_random_access(self): print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) def test_correctness(self): + raise SkipTest("Takes too long, enable it if you change the algorithm and want to be sure you decode packs correctly") pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" From cb4059bccea6d01e41d63c66fda502823a35220f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 22:07:37 +0200 Subject: [PATCH 0177/1392] Bumped version info to 0.5.3 --- doc/source/conf.py | 4 ++-- setup.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 28deb3106..723a34503 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'GitDB' -copyright = u'2010, Sebastian Thiel' +copyright = u'2011, Sebastian Thiel' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -47,7 +47,7 @@ # The short X.Y version. version = '0.5' # The full version, including alpha/beta/rc tags. -release = '0.5.1' +release = '0.5.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 3c6617422..86073971e 100755 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ def get_data_files(self): setup(cmdclass={'build_ext':build_ext_nofail}, name = "gitdb", - version = "0.5.2", + version = "0.5.3", description = "Git Object Database", author = "Sebastian Thiel", author_email = "byronimo@gmail.com", @@ -80,7 +80,7 @@ def get_data_files(self): ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('async (>=0.6.1)',), - install_requires='async >= 0.6.1', + requires=('async (>=0.6.1)', 'smmap (>=0.8.0)'), + install_requires=('async >= 0.6.1', 'smmap >= 0.8.0'), long_description = """GitDB is a pure-Python git object database""" ) From 09dd0eb9249493fc2d2897684035d753fc888acc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 10 Jun 2011 22:22:42 +0200 Subject: [PATCH 0178/1392] A tiny win32 fix, once again --- smmap/test/test_util.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 46de2ebaa..096c5f6df 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -71,9 +71,10 @@ def test_region(self): assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) # with the values we have, this test only works on windows where an alignment # size of 4096 is assumed. - if sys.platform == 'win32': - assert rhalfofs.includes_ofs(rofs) and rhalfofs.includes_ofs(0) - else: + # We only test on linux as it is inconsitent between the python versions + # as they use different mapping techniques to circumvent the missing offset + # argument of mmap. + if sys.platform != 'win32': assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) #END handle platforms From 84eedc5d1def7bfefefc729d09c39a6a9cde81f2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 14:54:17 +0200 Subject: [PATCH 0179/1392] Added option to help test cases to succeed on windows. Its for test systems only --- smmap/util.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/smmap/util.py b/smmap/util.py index 80ba3c5b9..7ced28991 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -99,7 +99,13 @@ class MapRegion(object): if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset #END handle additional slot - + + #{ Configuration + # Used for testing only. If True, all data will be loaded into memory at once. + # This makes sure no file handles will remain open. + _test_read_into_memory = False + #} END configuration + def __init__(self, path_or_fd, ofs, size, flags = 0): """Initialize a region, allocate the memory map @@ -132,7 +138,13 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! - self._mf = mmap(fd, min(os.fstat(fd).st_size - sizeofs, corrected_size), **kwargs) + actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) + if self._test_read_into_memory: + self._mf = self._read_into_memory(fd, ofs, actual_size) + else: + self._mf = mmap(fd, actual_size, **kwargs) + #END handle memory mode + self._size = len(self._mf) if self._need_compat_layer: @@ -144,6 +156,19 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): #END only close it if we opened it #END close file handle + def _read_into_memory(self, fd, offset, size): + """:return: string data as read from the given file descriptor, offset and size """ + os.lseek(fd, offset, os.SEEK_SET) + mf = '' + bytes_todo = size + while bytes_todo: + chunk = 1024*1024 + d = os.read(fd, chunk) + bytes_todo -= len(d) + mf += d + #END loop copy items + return mf + def __repr__(self): return "MapRegion<%i, %i>" % (self._b, self.size()) @@ -204,7 +229,7 @@ def includes_ofs(self, ofs): #} END interface - + class MapRegionList(list): """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( From 65dacbbd74a46698932cccdcab54f7558d1da169 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 20:23:55 +0200 Subject: [PATCH 0180/1392] Fixed up configuration to create api documentation for the code. Improved some markup to be valid for sphinx. --- doc/source/conf.py | 2 +- smmap/mman.py | 5 +++-- smmap/util.py | 4 +++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index a0dac1166..90409a138 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,7 +16,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.append(os.path.abspath('.')) +sys.path.append(os.path.abspath('../../')) # -- General configuration ----------------------------------------------------- diff --git a/smmap/mman.py b/smmap/mman.py index c6efc9496..9629eca46 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -354,8 +354,9 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): #{ Interface def make_cursor(self, path_or_fd): - """:return: a cursor pointing to the given path or file descriptor. - It can be used to map new regions of the file into memory + """ + :return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory :note: if a file descriptor is given, it is assumed to be open and valid, but may be closed afterwards. To refer to the same file, you may reuse your existing file descriptor, but keep in mind that new windows can only diff --git a/smmap/util.py b/smmap/util.py index 7ced28991..07bdf7997 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -20,7 +20,9 @@ #{ Utilities def align_to_mmap(num, round_up): - """Align the given integer number to the closest page offset, which usually is 4096 bytes. + """ + Align the given integer number to the closest page offset, which usually is 4096 bytes. + :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" From 78cdb214fb6273169433fa662ad630afc26eb36a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 21:07:03 +0200 Subject: [PATCH 0181/1392] Wrote introduction, readme.rst now points to the introduction page to keep things consistent --- README.rst | 52 +--------------------------- doc/source/api.rst | 42 +++++++++++++++++++++++ doc/source/index.rst | 3 ++ doc/source/intro.rst | 82 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 51 deletions(-) mode change 100644 => 120000 README.rst create mode 100644 doc/source/api.rst create mode 100644 doc/source/intro.rst diff --git a/README.rst b/README.rst deleted file mode 100644 index 4c22eed2b..000000000 --- a/README.rst +++ /dev/null @@ -1,51 +0,0 @@ -#################### -Sliding MMap (smmap) -#################### -A straight forward implementation of a slidinging memory map. -The idea is that every access to a file goes through a memory map manager, which will on demand map a region of a file and provide a string-like object for reading. - -When reading from it, you will have to check whether you are still within your window boundary, and possibly obtain a new window as required. - -The great benefit of this system is that you can use it to map files of any size even on 32 bit systems. Additionally it will be able to close unused windows right away to return system resources. If there are multiple clients for the same file and location, the same window will be reused as well. - -As there is a global management facility, you are also able to forcibly free all open handles which is handy on windows, which would otherwise prevent the deletion of the involved files. - -For convenience, a stream class is provided which hides the usage of the memory manager behind a simple stream interface. - -************ -LIMITATIONS -************ -* The access is readonly by design. -* In python below 2.6, memory maps will be created in compatability mode which works, but creates inefficient memory maps as they always start at offset 0. - -************ -REQUIREMENTS -************ -* runs Python 2.4 or higher, but needs Python 2.6 or higher to run properly as it needs the offset parameter of the mmap.mmap function. - -******* -Install -******* -TODO - -****** -Source -****** -The source is available at git://github.com/Byron/smmap.git and can be cloned using:: - - git clone git://github.com/Byron/smmap.git - -************ -MAILING LIST -************ -http://groups.google.com/group/git-python - -************* -ISSUE TRACKER -************* -https://github.com/Byron/smmap/issues - -******* -LICENSE -******* -New BSD License diff --git a/README.rst b/README.rst new file mode 120000 index 000000000..7cafde78d --- /dev/null +++ b/README.rst @@ -0,0 +1 @@ +doc/source/intro.rst \ No newline at end of file diff --git a/doc/source/api.rst b/doc/source/api.rst new file mode 100644 index 000000000..7e2854afa --- /dev/null +++ b/doc/source/api.rst @@ -0,0 +1,42 @@ +.. _api-label: + +############# +API Reference +############# + +**************** +smmap.mman +**************** + +.. automodule:: smmap.mman + :members: + :undoc-members: + +**************** +smmap.buf +**************** + +.. automodule:: smmap.buf + :members: + :undoc-members: + +**************** +smmap.exc +**************** + +.. automodule:: smmap.exc + :members: + :undoc-members: + +**************** +smmap.util +**************** + +.. automodule:: smmap.util + :members: + :undoc-members: + + + + + diff --git a/doc/source/index.rst b/doc/source/index.rst index cb03044e0..d25ef8244 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -12,6 +12,9 @@ Contents: .. toctree:: :maxdepth: 2 + intro + tutorial + api changes Indices and tables diff --git a/doc/source/intro.rst b/doc/source/intro.rst new file mode 100644 index 000000000..b35e78569 --- /dev/null +++ b/doc/source/intro.rst @@ -0,0 +1,82 @@ +########### +Motivation +########### +When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps. + +Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. + +######## +Overview +######## + +Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. + +To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. + +The interface also works around the missing offset parameter in python implementations up to python 2.5. + +Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. + +For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. + +############# +Prerequisites +############# +* Python 2.4, 2.5 or 2.6 +* OSX, Windows or Linux + +The package was tested on all of the previously mentioned configurations. + +########### +Limitations +########### +* The memory access is read-only by design. +* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. +* It wasn't tested on python 2.7 and 3.x. + +############### +Getting Started +############### +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + +################ +Installing smmap +################ +Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the `setuptools`_ or `pip`_ respectively:: + + $ easy_install smmap + # or + $ pip install smmap + +As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: + + $ python setup.py install + +################## +Homepage and Links +################## +The project is home on github at `https://github.com/Byron/smmap `_. + +The latest source can be cloned from github as well: + + * git://github.com/gitpython-developers/smmap.git + + +For support, please use the git-python mailing list: + + * http://groups.google.com/group/git-python + + +Issues can be filed on github: + + * https://github.com/Byron/smmap/issues + +################### +License Information +################### +*smmap* is licensed under the New BSD License. + +.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools +.. _pip: http://www.pip-installer.org/en/latest/ From a51b65d35d402791f774efe95d4b848cc524a403 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 22:48:55 +0200 Subject: [PATCH 0182/1392] Finished tutorial section, umproved capabilities of the buffer implementation to be more pythonic. Unfortunately, not all docs build yet because of some typical sphinx issue that results in an error which doesn't at all tell what the culprit actually is --- doc/source/intro.rst | 9 +-- doc/source/tutorial.rst | 118 ++++++++++++++++++++++++++++++++++++ smmap/buf.py | 10 +++ smmap/mman.py | 6 +- smmap/test/test_buf.py | 4 ++ smmap/test/test_tutorial.py | 83 +++++++++++++++++++++++++ 6 files changed, 221 insertions(+), 9 deletions(-) create mode 100644 doc/source/tutorial.rst create mode 100644 smmap/test/test_tutorial.py diff --git a/doc/source/intro.rst b/doc/source/intro.rst index b35e78569..30bff0ded 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -34,11 +34,6 @@ Limitations * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. * It wasn't tested on python 2.7 and 3.x. -############### -Getting Started -############### -It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. - ################ Installing smmap ################ @@ -53,7 +48,9 @@ As the command will install smmap in your respective python distribution, you wi If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: $ python setup.py install - + +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + ################## Homepage and Links ################## diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst new file mode 100644 index 000000000..917b24594 --- /dev/null +++ b/doc/source/tutorial.rst @@ -0,0 +1,118 @@ +.. _tutorial-label: + +########### +Usage Guide +########### +This text briefly introduces you to the basic design decisions and accompanying classes. + +****** +Design +****** +Per application, there is *MemoryManager* which is held as static instance and used throughout the application. It can be configured to keep your resources within certain limits. + +To access mapped regions, you require a cursor. Cursors point to exactly one file and serve as handles into it. As long as it exists, the respective memory region will remain available. + +For convenience, a buffer implementation is provided which handles cursors and resource allocation behind its simple buffer like interface. + +*************** +Memory Managers +*************** +There are two types of memory managers, one uses *static* windows, the other one uses *sliding* windows. A window is a region of a file mapped into memory. Although the names might be somewhat misleading as technically windows are always static, the *sliding* version will allocate relatively small windows whereas the *static* version will always map the whole file. + +The *static* manager does nothing more than keeping a client count on the respective memory maps which always map the whole file, which allows to make some assumptions that can lead to simplified data access and increased performance, but reduces the compatibility to 32 bit systems or giant files. + +The *sliding* memory manager therefore should be the default manager when preparing an application for handling huge amounts of data on 32 bit and 64 bit platforms:: + + import smmap + # This instance should be globally available in your application + # It is configured to be well suitable for 32-bit or 64 bit applications. + mman = smmap.SlidingWindowMapManager() + + # the manager provides much useful information about its current state + # like the amount of open file handles or the amount of mapped memory + mman.num_file_handles() + mman.mapped_memory_size() + # and many more ... + + +Cursors +******* +*Cursors* are handles that point onto a window, i.e. a region of a file mapped into memory. From them you may obtain a buffer through which the data of that window can actually be accessed:: + + import smmap.test.lib + fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size()-1] # last byte + + # its recommended not to create big slices when feeding the buffer + # into consumers (e.g. struct or zlib). + # Instead, either give the buffer directly, or use pythons buffer command. + buffer(c.buffer(), 1, 9) # first 9 bytes without copying them + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + +Now you would have to write your algorithms around this interface to properly slide through huge amounts of data. + +Alternatively you can use a convenience interface. + +******* +Buffers +******* +To make first use easier, at the expense of performance, there is a Buffer implementation which uses a cursor underneath. + +With it, you can access all data in a possibly huge file without having to take care of setting the cursor to different regions yourself:: + + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:]# access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset + + # it will stop using resources automatically once it goes out of scope + +Disadvantages +************* +Buffers cannot be used in place of strings or maps, hence you have to slice them to have valid input for the sorts of struct and zlib. A slice means a lot of data handling overhead which makes buffers slower compared to using cursors directly. + diff --git a/smmap/buf.py b/smmap/buf.py index c4d252251..9b2402687 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -47,6 +47,8 @@ def __len__(self): def __getitem__(self, i): c = self._c assert c.is_valid() + if i < 0: + i = self._size + i if not c.includes_ofs(i): c.use_region(i, 1) # END handle region usage @@ -57,6 +59,12 @@ def __getslice__(self, i, j): # fast path, slice fully included - safes a concatenate operation and # should be the default assert c.is_valid() + if i < 0: + i = self._size + i + if j == sys.maxint: + j = self._size + if j < 0: + j = self._size + j if (c.ofs_begin() <= i) and (j < c.ofs_end()): b = c.ofs_begin() return c.buffer()[i-b:j-b] @@ -68,6 +76,7 @@ def __getslice__(self, i, j): md = str() while l: c.use_region(ofs, l) + assert c.is_valid() d = c.buffer()[:l] ofs += len(d) l -= len(d) @@ -102,6 +111,7 @@ def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): self._size = size #END set size return res + # END use our cursor return False def end_access(self): diff --git a/smmap/mman.py b/smmap/mman.py index 9629eca46..deba99809 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -11,17 +11,16 @@ import sys from sys import getrefcount -__all__ = ["StaticWindowMapManager", "SlidingWindowMapManager"] +__all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"] #{ Utilities #}END utilities - class WindowCursor(object): """Pointer into the mapped region of the memory manager, keeping the map alive until it is destroyed and no other client uses it. - + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager :note: The current implementation is suited for static and sliding window managers, but it also means that it must be suited for the somewhat quite different sliding manager. It could be improved, but @@ -85,6 +84,7 @@ def assign(self, rhs): def use_region(self, offset = 0, size = 0, flags = 0): """Assure we point to a window which allows access to the given offset into the file + :param offset: absolute offset in bytes into the file :param size: amount of bytes to map. If 0, all available bytes will be mapped :param flags: additional flags to be given to os.open in case a file handle is initially opened diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d8b7fbcab..9881c6294 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -50,6 +50,10 @@ def test_basics(self): assert data[offset] == buf[0] assert data[offset:offset*2] == buf[0:offset] + # negative indices, partial slices + assert buf[-1] == buf[len(buf)-1] + assert buf[-10:] == buf[len(buf)-10:len(buf)] + # end access makes its cursor invalid buf.end_access() assert not buf.cursor().is_valid() diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py new file mode 100644 index 000000000..a9f4b1c08 --- /dev/null +++ b/smmap/test/test_tutorial.py @@ -0,0 +1,83 @@ +from lib import TestBase + +class TestTutorial(TestBase): + + def test_example(self): + # Memory Managers + ################## + import smmap + # This instance should be globally available in your application + # It is configured to be well suitable for 32-bit or 64 bit applications. + mman = smmap.SlidingWindowMapManager() + + # the manager provides much useful information about its current state + # like the amount of open file handles or the amount of mapped memory + assert mman.num_file_handles() == 0 + assert mman.mapped_memory_size() == 0 + # and many more ... + + # Cursors + ########## + import smmap.test.lib + fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size()-1] # last byte + + # its recommended not to create big slices when feeding the buffer + # into consumers (e.g. struct or zlib). + # Instead, either give the buffer directly, or use pythons buffer command. + buffer(c.buffer(), 1, 9) # first 9 bytes without copying them + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + # Buffers + ######### + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:]# access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset + + # it will stop using resources automatically once it goes out of scope + From ec511365bc641a320c66ce4e796918e58a1567c8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 22:56:16 +0200 Subject: [PATCH 0183/1392] It turned out the :note: docstring was not supported. Now all documentation is being generated --- doc/source/api.rst | 24 ++++++++++++------------ smmap/mman.py | 39 ++++++++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/doc/source/api.rst b/doc/source/api.rst index 7e2854afa..cddd268c4 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -4,33 +4,33 @@ API Reference ############# -**************** -smmap.mman -**************** +*********************** +Mapped Memory Managers +*********************** .. automodule:: smmap.mman :members: :undoc-members: -**************** -smmap.buf -**************** +******* +Buffers +******* .. automodule:: smmap.buf :members: :undoc-members: -**************** -smmap.exc -**************** +********** +Exceptions +********** .. automodule:: smmap.exc :members: :undoc-members: -**************** -smmap.util -**************** +********* +Utilities +********* .. automodule:: smmap.util :members: diff --git a/smmap/mman.py b/smmap/mman.py index deba99809..15bb012b4 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -18,13 +18,15 @@ class WindowCursor(object): - """Pointer into the mapped region of the memory manager, keeping the map + """ + Pointer into the mapped region of the memory manager, keeping the map alive until it is destroyed and no other client uses it. Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager - :note: The current implementation is suited for static and sliding window managers, but it also means - that it must be suited for the somewhat quite different sliding manager. It could be improved, but - I see no real need to do so.""" + + **Note**: The current implementation is suited for static and sliding window managers, but it also means + that it must be suited for the somewhat quite different sliding manager. It could be improved, but + I see no real need to do so.""" __slots__ = ( '_manager', # the manger keeping all file regions '_rlist', # a regions list with regions for our file @@ -91,8 +93,9 @@ def use_region(self, offset = 0, size = 0, flags = 0): for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. This is not the case if the mapping failed becaues we reached the end of the file - :note: The size actually mapped may be smaller than the given size. If that is the case, - either the file has reached its end, or the map was created between two existing regions""" + + **note**: The size actually mapped may be smaller than the given size. If that is the case, + either the file has reached its end, or the map was created between two existing regions""" need_region = True man = self._manager fsize = self._rlist.file_size() @@ -123,9 +126,10 @@ def use_region(self, offset = 0, size = 0, flags = 0): def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region - :note: the cursor unuses the region automatically upon destruction. It is recommended - to unuse the region once you are done reading from it in persistent cursors as it - helps to free up resource more quickly""" + + **note** the cursor unuses the region automatically upon destruction. It is recommended + to unuse the region once you are done reading from it in persistent cursors as it + helps to free up resource more quickly""" self._region = None # note: should reset ofs and size, but we spare that for performance. Its not # allowed to query information if we are not valid ! @@ -133,9 +137,11 @@ def unuse_region(self): def buffer(self): """Return a buffer object which allows access to our memory region from our offset to the window size. Please note that it might be smaller than you requested when calling use_region() - :note: You can only obtain a buffer if this instance is_valid() ! - :note: buffers should not be cached passed the duration of your access as it will - prevent resources from being freed even though they might not be accounted for anymore !""" + + **note** You can only obtain a buffer if this instance is_valid() ! + + **note** buffers should not be cached passed the duration of your access as it will + prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.buffer(), self._ofs, self._size) def map(self): @@ -155,7 +161,8 @@ def is_associated(self): def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor - :note: only if is_valid() is True""" + + **note** only if is_valid() is True""" return self._region._b + self._ofs def ofs_end(self): @@ -177,7 +184,8 @@ def region_ref(self): def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors current region - :note: cursor must be valid for this to work""" + + **note** cursor must be valid for this to work""" # unroll methods return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) @@ -199,7 +207,8 @@ def path(self): def fd(self): """:return: file descriptor used to create the underlying mapping. - :note: it is not required to be valid anymore + + **note** it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), basestring): raise ValueError("File descriptor queried although mapping was generated from path") From 59cd3da62b5038783deeb2883262682758ec1eec Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 22:58:44 +0200 Subject: [PATCH 0184/1392] Made README.rst a copy of intro.rst. unfortunately symlinks are not followed by github. This is a real issue to me ... --- README.rst | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) mode change 120000 => 100644 README.rst diff --git a/README.rst b/README.rst deleted file mode 120000 index 7cafde78d..000000000 --- a/README.rst +++ /dev/null @@ -1 +0,0 @@ -doc/source/intro.rst \ No newline at end of file diff --git a/README.rst b/README.rst new file mode 100644 index 000000000..30bff0ded --- /dev/null +++ b/README.rst @@ -0,0 +1,79 @@ +########### +Motivation +########### +When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps. + +Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. + +######## +Overview +######## + +Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. + +To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. + +The interface also works around the missing offset parameter in python implementations up to python 2.5. + +Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. + +For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. + +############# +Prerequisites +############# +* Python 2.4, 2.5 or 2.6 +* OSX, Windows or Linux + +The package was tested on all of the previously mentioned configurations. + +########### +Limitations +########### +* The memory access is read-only by design. +* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. +* It wasn't tested on python 2.7 and 3.x. + +################ +Installing smmap +################ +Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the `setuptools`_ or `pip`_ respectively:: + + $ easy_install smmap + # or + $ pip install smmap + +As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. + +If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: + + $ python setup.py install + +It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. + +################## +Homepage and Links +################## +The project is home on github at `https://github.com/Byron/smmap `_. + +The latest source can be cloned from github as well: + + * git://github.com/gitpython-developers/smmap.git + + +For support, please use the git-python mailing list: + + * http://groups.google.com/group/git-python + + +Issues can be filed on github: + + * https://github.com/Byron/smmap/issues + +################### +License Information +################### +*smmap* is licensed under the New BSD License. + +.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools +.. _pip: http://www.pip-installer.org/en/latest/ From f097bd611a82289d6bb95074fdf596332cb1c980 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 23:10:08 +0200 Subject: [PATCH 0185/1392] Fixed wrong operating system fields --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e2bb622fd..2e97e59c6 100755 --- a/setup.py +++ b/setup.py @@ -41,8 +41,8 @@ "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Operating System :: POSIX", - "Operating System :: Windows", - "Operating System :: OSX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", ], long_description=long_description, From cf297b7b81bc5f6011c49d818d776ed7915fa1ee Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 23:33:49 +0200 Subject: [PATCH 0186/1392] Removed possibly invalid documentation tags --- smmap/buf.py | 6 +++--- smmap/mman.py | 53 +++++++++++++++++++++++++++++---------------------- smmap/util.py | 3 ++- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 9b2402687..00ddbacd9 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -12,9 +12,9 @@ class SlidingWindowMapBuffer(object): The buffer is relative, that is if you map an offset, index 0 will map to the first byte at the offset you used during initialization or begin_access - :note: Although this type effectively hides the fact that there are mapped windows - underneath, it can unfortunately not be used in any non-pure python method which - needs a buffer or string""" + **Note:** Although this type effectively hides the fact that there are mapped windows + underneath, it can unfortunately not be used in any non-pure python method which + needs a buffer or string""" __slots__ = ( '_c', # our cursor '_size', # our supposed size diff --git a/smmap/mman.py b/smmap/mman.py index 15bb012b4..9b08ae969 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -24,7 +24,7 @@ class WindowCursor(object): Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager - **Note**: The current implementation is suited for static and sliding window managers, but it also means + **Note:**: The current implementation is suited for static and sliding window managers, but it also means that it must be suited for the somewhat quite different sliding manager. It could be improved, but I see no real need to do so.""" __slots__ = ( @@ -94,7 +94,7 @@ def use_region(self, offset = 0, size = 0, flags = 0): :return: this instance - it should be queried for whether it points to a valid memory region. This is not the case if the mapping failed becaues we reached the end of the file - **note**: The size actually mapped may be smaller than the given size. If that is the case, + **Note:**: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" need_region = True man = self._manager @@ -127,7 +127,7 @@ def use_region(self, offset = 0, size = 0, flags = 0): def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region - **note** the cursor unuses the region automatically upon destruction. It is recommended + **Note:** the cursor unuses the region automatically upon destruction. It is recommended to unuse the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None @@ -138,9 +138,9 @@ def buffer(self): """Return a buffer object which allows access to our memory region from our offset to the window size. Please note that it might be smaller than you requested when calling use_region() - **note** You can only obtain a buffer if this instance is_valid() ! + **Note:** You can only obtain a buffer if this instance is_valid() ! - **note** buffers should not be cached passed the duration of your access as it will + **Note:** buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.buffer(), self._ofs, self._size) @@ -162,7 +162,7 @@ def is_associated(self): def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor - **note** only if is_valid() is True""" + **Note:** only if is_valid() is True""" return self._region._b + self._ofs def ofs_end(self): @@ -185,7 +185,7 @@ def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors current region - **note** cursor must be valid for this to work""" + **Note:** cursor must be valid for this to work""" # unroll methods return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) @@ -208,7 +208,7 @@ def path(self): def fd(self): """:return: file descriptor used to create the underlying mapping. - **note** it is not required to be valid anymore + **Note:** it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), basestring): raise ValueError("File descriptor queried although mapping was generated from path") @@ -289,9 +289,11 @@ def _collect_lru_region(self, size): :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region :return: Amount of freed regions - :note: We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. - If the system runs out of memory, it will tell. - :todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + + **Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. + If the system runs out of memory, it will tell. + + **todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): lru_region = None @@ -366,15 +368,18 @@ def make_cursor(self, path_or_fd): """ :return: a cursor pointing to the given path or file descriptor. It can be used to map new regions of the file into memory - :note: if a file descriptor is given, it is assumed to be open and valid, - but may be closed afterwards. To refer to the same file, you may reuse - your existing file descriptor, but keep in mind that new windows can only - be mapped as long as it stays valid. This is why the using actual file paths - are preferred unless you plan to keep the file descriptor open. - :note: file descriptors are problematic as they are not necessarily unique, as two - different files opened and closed in succession might have the same file descriptor id. - :note: Using file descriptors directly is faster once new windows are mapped as it - prevents the file to be opened again just for the purpose of mapping it.""" + + **Note:** if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open. + + **Note:** file descriptors are problematic as they are not necessarily unique, as two + different files opened and closed in succession might have the same file descriptor id. + + **Note:** Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) if regions is None: regions = self.MapRegionListCls(path_or_fd) @@ -426,7 +431,8 @@ def force_map_handle_removal_win(self, base_path): This really may only be used if you know that the items which keep the cursors alive will not be using it anymore. They need to be recreated ! :return: Amount of closed handles - :note: does nothing on non-windows platforms""" + + **Note:** does nothing on non-windows platforms""" if sys.platform != 'win32': return #END early bailout @@ -451,8 +457,9 @@ class SlidingWindowMapManager(StaticWindowMapManager): which result from each mmap call, the least recently used, and currently unused mapped regions are unloaded automatically. - :note: currently not thread-safe ! - :note: in the current implementation, we will automatically unload windows if we either cannot + **Note:** currently not thread-safe ! + + **Note:** in the current implementation, we will automatically unload windows if we either cannot create more memory maps (as the open file handles limit is hit) or if we have allocated more than a safe amount of memory already, which would possibly cause memory allocations to fail as our address space is full.""" diff --git a/smmap/util.py b/smmap/util.py index 07bdf7997..b0fd83b3f 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -88,7 +88,8 @@ def extend_right_to(self, window, max_size): class MapRegion(object): """Defines a mapped region of memory, aligned to pagesizes - :note: deallocates used region automatically on destruction""" + + **Note:** deallocates used region automatically on destruction""" __slots__ = [ '_b' , # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) From 4524faf0d0c5383268b134084954b34faeaa766d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 13 Jun 2011 23:29:22 +0200 Subject: [PATCH 0187/1392] Fixed up docs for upcoming release. Bumped version to 0.5.3 --- Makefile | 2 +- gitdb/__init__.py | 7 +++++++ gitdb/db/base.py | 7 ++++--- gitdb/db/mem.py | 4 ++-- gitdb/db/pack.py | 2 +- gitdb/ext/smmap | 2 +- gitdb/fun.py | 15 +++++++++------ gitdb/pack.py | 29 +++++++++++++++++------------ gitdb/stream.py | 6 +++--- gitdb/test/db/lib.py | 2 +- gitdb/util.py | 28 ++++++++++++++++++---------- setup.py | 9 +++++---- 12 files changed, 69 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index e65c55a6d..c6c159bdc 100644 --- a/Makefile +++ b/Makefile @@ -23,5 +23,5 @@ clean:: rm -f *.so coverage:: build - PYTHONPATH=. $(PYTHON) $(TESTRUNNER) --cover-package=dulwich --with-coverage --cover-erase --cover-inclusive gitdb + PYTHONPATH=. $(PYTHON) $(TESTRUNNER) --cover-package=gitdb --with-coverage --cover-erase --cover-inclusive gitdb diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 775c969cf..91359105c 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -24,6 +24,13 @@ def _init_externals(): _init_externals() +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/gitpython-developers/gitdb" +version_info = (0, 5, 3) +__version__ = '.'.join(str(i) for i in version_info) + + # default imports from db import * from base import * diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 2189d4193..984acafbf 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -72,7 +72,8 @@ def stream_async(self, reader): :param reader: see ``info`` :param max_threads: see ``ObjectDBW.store`` :return: async.Reader yielding OStream|InvalidOStream instances in any order - :note: depending on the system configuration, it might not be possible to + + **Note:** depending on the system configuration, it might not be possible to read all OStreams at once. Instead, read them individually using reader.read(x) where x is small enough.""" # base implementation just uses the stream method repeatedly @@ -140,7 +141,7 @@ def store_async(self, reader): The same instances will be used in the output channel as were received in by the Reader. - :note:As some ODB implementations implement this operation atomic, they might + **Note:** As some ODB implementations implement this operation atomic, they might abort the whole operation if one item could not be processed. Hence check how many items have actually been produced.""" # base implementation uses store to perform the work @@ -158,7 +159,7 @@ def __init__(self, root_path): """Initialize this instance to look for its files at the given root path All subsequent operations will be relative to this path :raise InvalidDBRoot: - :note: The base will not perform any accessablity checking as the base + **Note:** The base will not perform any accessablity checking as the base might not yet be accessible, but become accessible before the first access.""" super(FileDBBase, self).__init__() diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 8012ad15e..5d76c83cc 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -33,7 +33,7 @@ class MemoryDB(ObjectDBR, ObjectDBW): it to the actual physical storage, as it allows to query whether object already exists in the target storage before introducing actual IO - :note: memory is currently not threadsafe, hence the async methods cannot be used + **Note:** memory is currently not threadsafe, hence the async methods cannot be used for storing""" def __init__(self): @@ -92,7 +92,7 @@ def sha_iter(self): def stream_copy(self, sha_iter, odb): """Copy the streams as identified by sha's yielded by sha_iter into the given odb The streams will be copied directly - :note: the object will only be written if it did not exist in the target db + **Note:** the object will only be written if it did not exist in the target db :return: amount of streams actually copied into odb. If smaller than the amount of input shas, one or more objects did already exist in odb""" count = 0 diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index eef3f712e..4c9d0b919 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -58,7 +58,7 @@ def _pack_info(self, sha): """:return: tuple(entity, index) for an item at the given sha :param sha: 20 or 40 byte sha :raise BadObject: - :note: This method is not thread-safe, but may be hit in multi-threaded + **Note:** This method is not thread-safe, but may be hit in multi-threaded operation. The worst thing that can happen though is a counter that was not incremented, or the list being in wrong order. So we safe the time for locking here, lets see how that goes""" diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 84eedc5d1..f097bd611 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 84eedc5d1def7bfefefc729d09c39a6a9cde81f2 +Subproject commit f097bd611a82289d6bb95074fdf596332cb1c980 diff --git a/gitdb/fun.py b/gitdb/fun.py index 5bbe8efc3..66130ebee 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -138,7 +138,7 @@ def _closest_index(dcl, absofs): """:return: index at which the given absofs should be inserted. The index points to the DeltaChunk with a target buffer absofs that equals or is greater than absofs. - :note: global method for performance only, it belongs to DeltaChunkList""" + **Note:** global method for performance only, it belongs to DeltaChunkList""" lo = 0 hi = len(dcl) while lo < hi: @@ -414,9 +414,11 @@ def pack_object_header_info(data): return (type_id, size, i) def create_pack_object_header(obj_type, obj_size): - """:return: string defining the pack header comprised of the object type - and its incompressed size in bytes - :parmam obj_type: pack type_id of the object + """ + :return: string defining the pack header comprised of the object type + and its incompressed size in bytes + + :param obj_type: pack type_id of the object :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte hdr = str() # output string @@ -483,7 +485,7 @@ def stream_copy(read, write, size, chunk_size): Copy a stream up to size bytes using the provided read and write methods, in chunks of chunk_size - :note: its much like stream_copy utility, but operates just using methods""" + **Note:** its much like stream_copy utility, but operates just using methods""" dbw = 0 # num data bytes written # WRITE ALL DATA UP TO SIZE @@ -597,7 +599,8 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data :param write: write method taking a chunk of bytes - :note: transcribed to python from the similar routine in patch-delta.c""" + + **Note:** transcribed to python from the similar routine in patch-delta.c""" i = 0 db = delta_buf while i < delta_buf_size: diff --git a/gitdb/pack.py b/gitdb/pack.py index 0679a6ecf..d840441e9 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -173,7 +173,7 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): class IndexWriter(object): """Utility to cache index information, allowing to write all information later in one go to the given stream - :note: currently only writes v2 indices""" + **Note:** currently only writes v2 indices""" __slots__ = '_objs' def __init__(self): @@ -391,7 +391,8 @@ def indexfile_checksum(self): def offsets(self): """:return: sequence of all offsets in the order in which they were written - :note: return value can be random accessed, but may be immmutable""" + + **Note:** return value can be random accessed, but may be immmutable""" if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears @@ -497,10 +498,10 @@ class PackFile(LazyMixin): packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be fine though. - :note: at some point, this might be implemented using streams as well, or - streams are an alternate path in the case memory maps cannot be created - for some reason - one clearly doesn't want to read 10GB at once in that - case""" + **Note:** at some point, this might be implemented using streams as well, or + streams are an alternate path in the case memory maps cannot be created + for some reason - one clearly doesn't want to read 10GB at once in that + case""" __slots__ = ('_packpath', '_cursor', '_size', '_version') pack_signature = 0x5041434b # 'PACK' @@ -625,8 +626,9 @@ def stream_iter(self, start_offset=0): to access the data in the pack directly. :param start_offset: offset to the first object to iterate. If 0, iteration starts at the very first object in the pack. - :note: Iterating a pack directly is costly as the datastream has to be decompressed - to determine the bounds between the objects""" + + **Note:** Iterating a pack directly is costly as the datastream has to be decompressed + to determine the bounds between the objects""" return self._iter_objects(start_offset, as_stream=True) #} END Read-Database like Interface @@ -902,9 +904,11 @@ def write_pack(cls, object_iter, pack_write, index_write=None, :param zlib_compression: the zlib compression level to use :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack and over all contents of the index. If index_write was None, index_binsha will be None - :note: The destination of the write functions is up to the user. It could - be a socket, or a file for instance - :note: writes only undeltified objects""" + + **Note:** The destination of the write functions is up to the user. It could + be a socket, or a file for instance + + **Note:** writes only undeltified objects""" objs = object_iter if not object_count: if not isinstance(object_iter, (tuple, list)): @@ -979,7 +983,8 @@ def create(cls, object_iter, base_dir, object_count = None, zlib_compression = z and corresponding index file. The pack contains all OStream objects contained in object iter. :param base_dir: directory which is to contain the files :return: PackEntity instance initialized with the new pack - :note: for more information on the other parameters see the write_pack method""" + + **Note:** for more information on the other parameters see the write_pack method""" pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) pack_write = lambda d: os.write(pack_fd, d) diff --git a/gitdb/stream.py b/gitdb/stream.py index 8010a0551..632213c27 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -51,7 +51,7 @@ class DecompressMemMapReader(LazyMixin): To read efficiently, you clearly don't want to read individual bytes, instead, read a few kilobytes at least. - :note: The chunk-size should be carefully selected as it will involve quite a bit + **Note:** The chunk-size should be carefully selected as it will involve quite a bit of string copying due to the way the zlib is implemented. Its very wasteful, hence we try to find a good tradeoff between allocation time and number of times we actually allocate. An own zlib implementation would be good here @@ -609,8 +609,8 @@ class FDCompressedSha1Writer(Sha1Writer): """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor - :note: operates on raw file descriptors - :note: for this to work, you have to use the close-method of this instance""" + **Note:** operates on raw file descriptors + **Note:** for this to work, you have to use the close-method of this instance""" __slots__ = ("fd", "sha1", "zip") # default exception diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 416c8c588..4af4483c7 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -65,7 +65,7 @@ def _assert_object_writing_simple(self, db): def _assert_object_writing(self, db): """General tests to verify object writing, compatible to ObjectDBW - :note: requires write access to the database""" + **Note:** requires write access to the database""" # start in 'dry-run' mode, using a simple sha1 writer ostreams = (ZippedStoreShaWriter, None) for ostreamcls in ostreams: diff --git a/gitdb/util.py b/gitdb/util.py index 4ce615585..23784de3f 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -119,8 +119,9 @@ def __getslice__(self, start, end): #{ Routines def make_sha(source=''): - """A python2.4 workaround for the sha/hashlib module fiasco - :note: From the dulwich project """ + """A python2.4 workaround for the sha/hashlib module fiasco + + **Note** From the dulwich project """ try: return hashlib.sha1(source) except NameError: @@ -146,6 +147,7 @@ def allocate_memory(size): def file_contents_ro(fd, stream=False, allow_mmap=True): """:return: read-only contents of the file represented by the file descriptor fd + :param fd: file descriptor opened for reading :param stream: if False, random access is provided, otherwise the stream interface is provided. @@ -173,14 +175,16 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): """Get the file contents at filepath as fast as possible + :return: random access compatible memory of the given filepath :param stream: see ``file_contents_ro`` :param allow_mmap: see ``file_contents_ro`` :param flags: additional flags to pass to os.open :raise OSError: If the file could not be opened - :note: for now we don't try to use O_NOATIME directly as the right value needs to be - shared per database in fact. It only makes a real difference for loose object - databases anyway, and they use it with the help of the ``flags`` parameter""" + + **Note** for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object + databases anyway, and they use it with the help of the ``flags`` parameter""" fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) try: return file_contents_ro(fd, stream, allow_mmap) @@ -189,7 +193,8 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): # END assure file is closed def sliding_ro_buffer(filepath, flags=0): - """:return: a buffer compatible object which uses our mapped memory manager internally + """ + :return: a buffer compatible object which uses our mapped memory manager internally ready to read the whole given filepath""" return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) @@ -254,7 +259,7 @@ class LockedFD(object): This type handles error correctly in that it will assure a consistent state on destruction. - :note: with this setup, parallel reading is not possible""" + **note** with this setup, parallel reading is not possible""" __slots__ = ("_filepath", '_fd', '_write') def __init__(self, filepath): @@ -283,7 +288,8 @@ def open(self, write=False, stream=False): and must not be closed directly :raise IOError: if the lock could not be retrieved :raise OSError: If the actual file could not be opened for reading - :note: must only be called once""" + + **note** must only be called once""" if self._write is not None: raise AssertionError("Called %s multiple times" % self.open) @@ -327,13 +333,15 @@ def commit(self): """When done writing, call this function to commit your changes into the actual file. The file descriptor will be closed, and the lockfile handled. - :note: can be called multiple times""" + + **Note** can be called multiple times""" self._end_writing(successful=True) def rollback(self): """Abort your operation without any changes. The file descriptor will be closed, and the lock released. - :note: can be called multiple times""" + + **Note** can be called multiple times""" self._end_writing(successful=False) def _end_writing(self, successful=True): diff --git a/setup.py b/setup.py index 86073971e..b5c8c7046 100755 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ from distutils.command.build_ext import build_ext import os, sys +import gitdb as meta # wow, this is a mixed bag ... I am pretty upset about all of this ... setuptools_build_py_module = None @@ -69,11 +70,11 @@ def get_data_files(self): setup(cmdclass={'build_ext':build_ext_nofail}, name = "gitdb", - version = "0.5.3", + version = meta.__version__, description = "Git Object Database", - author = "Sebastian Thiel", - author_email = "byronimo@gmail.com", - url = "http://gitorious.org/git-python/gitdb", + author = meta.__author__, + author_email = meta.__contact__, + url = meta.__homepage__, packages = ('gitdb', 'gitdb.db', 'gitdb.test', 'gitdb.test.db', 'gitdb.test.performance'), package_data={ 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, package_dir = {'gitdb':'gitdb'}, From a5ed410aa0d3bed587214c3c017af2916b740da1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 3 Jul 2011 13:39:19 +0200 Subject: [PATCH 0188/1392] removed test suite from being distributed. It didn't work properly anyway and I am not going to dig into the setup tools mess --- setup.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index b5c8c7046..7924b5a07 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ from distutils.command.build_ext import build_ext import os, sys -import gitdb as meta # wow, this is a mixed bag ... I am pretty upset about all of this ... setuptools_build_py_module = None @@ -68,15 +67,23 @@ def get_data_files(self): setuptools_build_py_module.build_py._get_data_files = get_data_files # END apply setuptools patch too +# NOTE: This is currently duplicated from the gitdb.__init__ module, as we cannot +# satisfy the dependencies at installation time, unfortunately, due to inherent limitations +# of distutils, which cannot install the prerequesites of a package before the acutal package. +__author__ = "Sebastian Thiel" +__contact__ = "byronimo@gmail.com" +__homepage__ = "https://github.com/gitpython-developers/gitdb" +version_info = (0, 5, 3) +__version__ = '.'.join(str(i) for i in version_info) + setup(cmdclass={'build_ext':build_ext_nofail}, name = "gitdb", - version = meta.__version__, + version = __version__, description = "Git Object Database", - author = meta.__author__, - author_email = meta.__contact__, - url = meta.__homepage__, - packages = ('gitdb', 'gitdb.db', 'gitdb.test', 'gitdb.test.db', 'gitdb.test.performance'), - package_data={ 'gitdb.test' : ['fixtures/packs/*', 'fixtures/objects/7b/*']}, + author = __author__, + author_email = __contact__, + url = __homepage__, + packages = ('gitdb', 'gitdb.db'), package_dir = {'gitdb':'gitdb'}, ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", From aea587d9b414d7f150922c2923a1b9394d0d0543 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 08:58:22 +0200 Subject: [PATCH 0189/1392] Added license info for packs --- LICENSE | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/LICENSE b/LICENSE index be11e73c1..0d6fe8bdb 100644 --- a/LICENSE +++ b/LICENSE @@ -28,3 +28,15 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Additional Licenses +------------------- +The files at +gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx +and +gitdb/test/fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack +are licensed under GNU GPL as part of the git source repository, +see http://en.wikipedia.org/wiki/Git_%28software%29 for more information. + +They are not required for the actual operation, which is why they are not found +in the distribution package. From 9c3eb3dafd765ee2e8299b53a1d8d780d9a8f55b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 16:36:30 +0200 Subject: [PATCH 0190/1392] Fixed possible bug as a method was called using an old signature. Apparently this code branch never ran in the tests --- smmap/mman.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 9b08ae969..ef9d43df0 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -68,7 +68,7 @@ def _copy_from(self, rhs): self._size = rhs._size if self._region is not None: - self._region.increment_usage_count(1) + self._region.increment_usage_count() # END handle regions def __copy__(self): @@ -358,7 +358,6 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # END handle array assert r.includes_ofs(offset) - #assert r.includes_ofs(offset+size-1) return r #}END internal methods From 0e64168dd3f43b02857e60183d40c86480f01dc7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 15:03:16 +0200 Subject: [PATCH 0191/1392] pack: updated to use its cursor properly, which will be required if huge packs should be handled. This reduces performance as each access requires the windows to be checked/adjusted, but that is how it is. This should be circumvented using other backends, like the one of the gitcmd or libgit2. Default is now the sliding memory map manager --- gitdb/pack.py | 32 +++++++++++++++++++------------- gitdb/util.py | 3 ++- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index d840441e9..c6d1cc313 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -73,7 +73,7 @@ #{ Utilities -def pack_object_at(data, offset, as_stream): +def pack_object_at(cursor, offset, as_stream): """ :return: Tuple(abs_data_offset, PackInfo|PackStream) an object of the correct type according to the type_id of the object. @@ -83,7 +83,7 @@ def pack_object_at(data, offset, as_stream): :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" - data = buffer(data, offset) + data = cursor.use_region(offset).buffer() type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins delta_info = None @@ -269,6 +269,10 @@ def _set_cache_(self, attr): # that we can actually write to the location - it could be a read-only # alternate for instance self._cursor = mman.make_cursor(self._indexpath).use_region() + # We will assume that the index will always fully fit into memory ! + if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): + raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (self._indexpath, self._cursor.file_size(), mman.window_size())) + #END assert window size else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties @@ -528,13 +532,13 @@ def _set_cache_(self, attr): def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" - data = self._cursor.map() - content_size = len(data) - self.footer_size + c = self._cursor + content_size = c.file_size() - self.footer_size cur_offset = start_offset or self.first_object_offset null = NullStream() while cur_offset < content_size: - data_offset, ostream = pack_object_at(data, cur_offset, True) + data_offset, ostream = pack_object_at(c, cur_offset, True) # scrub the stream to the end - this decompresses the object, but yields # the amount of compressed bytes we need to get to the next offset @@ -563,12 +567,14 @@ def version(self): def data(self): """ :return: read-only data of this pack. It provides random access and usually - is a memory map""" - return self._cursor.map() + is a memory map. + :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" + # can use map as we are starting at offset 0. Otherwise we would have to use buffer() + return self._cursor.use_region().map() def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" - return self._cursor.map()[-20:] + return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] def path(self): """:return: path to the packfile""" @@ -587,9 +593,9 @@ def collect_streams(self, offset): If the object at offset is no delta, the size of the list is 1. :param offset: specifies the first byte of the object within this pack""" out = list() - data = self._cursor.map() + c = self._cursor while True: - ostream = pack_object_at(data, offset, True)[1] + ostream = pack_object_at(c, offset, True)[1] out.append(ostream) if ostream.type_id == OFS_DELTA: offset = ostream.pack_offset - ostream.delta_info @@ -611,14 +617,14 @@ def info(self, offset): :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._cursor.map(), offset or self.first_object_offset, False)[1] + return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._cursor.map(), offset or self.first_object_offset, True)[1] + return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] def stream_iter(self, start_offset=0): """ @@ -702,7 +708,7 @@ def _object(self, sha, as_stream, index=-1): sha = self._index.sha(index) # END assure sha is present ( in output ) offset = self._index.offset(index) - type_id, uncomp_size, data_rela_offset = pack_object_header_info(buffer(self._pack._cursor.map(), offset)) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(self._pack._cursor.use_region(offset).buffer()) if as_stream: if type_id not in delta_types: packstream = self._pack.stream(offset) diff --git a/gitdb/util.py b/gitdb/util.py index 23784de3f..e96c133e8 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -25,12 +25,13 @@ from async import ThreadPool from smmap import ( StaticWindowMapManager, + SlidingWindowMapManager, SlidingWindowMapBuffer ) # initialize our global memory manager instance # Use it to free cached (and unused) resources. -mman = StaticWindowMapManager() +mman = SlidingWindowMapManager() try: import hashlib From ef5dc3d968b3aeed16a02ec705f89b72ad46fa84 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 16:40:54 +0200 Subject: [PATCH 0192/1392] Optimized test_pack_streaming not to cache the objects anymore. Instead an iterator is provided which does the job. Previously it would easily use 750 MB of ram to keep all the associated objects, more than 350k. Still a lot of memory for just 350k objects, but its python after all --- gitdb/ext/smmap | 2 +- gitdb/test/performance/test_pack_streaming.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index f097bd611..9c3eb3daf 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit f097bd611a82289d6bb95074fdf596332cb1c980 +Subproject commit 9c3eb3dafd765ee2e8299b53a1d8d780d9a8f55b diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 795ed1e26..3c40ed0fb 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -40,10 +40,9 @@ def test_pack_writing(self): count = 0 total_size = 0 st = time() - objs = list() for sha in pdb.sha_iter(): count += 1 - objs.append(pdb.stream(sha)) + pdb.stream(sha) if count == ni: break #END gather objects for pack-writing @@ -51,7 +50,7 @@ def test_pack_writing(self): print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) st = time() - PackEntity.write_pack(objs, ostream.write) + PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) elapsed = time() - st total_kb = ostream.bytes_written() / 1000 print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) From a4deb8461e6fbca0306a24f22b0c494679ad4757 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 16:45:39 +0200 Subject: [PATCH 0193/1392] wrote change log for next release. Choosing memory manager type based on the actual python version for best efficiency --- doc/source/changes.rst | 5 +++++ gitdb/util.py | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 999cc1309..839bf16a8 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,11 @@ Changelog ######### +***** +0.5.4 +***** +* Adjusted implementation to use the SlidingMemoryManager by default in python 2.6 for efficiency reasons. In Python 2.4, the StaticMemoryManager will be used instead. + ***** 0.5.3 ***** diff --git a/gitdb/util.py b/gitdb/util.py index e96c133e8..013f5fc78 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -31,7 +31,11 @@ # initialize our global memory manager instance # Use it to free cached (and unused) resources. -mman = SlidingWindowMapManager() +if sys.version_info[1] < 6: + mman = StaticWindowMapManager() +else: + mman = SlidingWindowMapManager() +#END handle mman try: import hashlib From d13e3aeb168645965720ddec1f469e05771563ef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 16:55:32 +0200 Subject: [PATCH 0194/1392] updated changelog, bumped version --- doc/source/changes.rst | 6 ++++++ smmap/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index ee17e0af0..03148fb31 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +********** +v0.8.1 +********** +- A single bugfix + + ********** v0.8.0 ********** diff --git a/smmap/__init__.py b/smmap/__init__.py index 769858fa5..ae6e72eba 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 0) +version_info = (0, 8, 1) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 656a2e0b4da7d60ac638d1615751a89efb3a4eee Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 Jul 2011 17:00:27 +0200 Subject: [PATCH 0195/1392] bumped version to 0.5.4 --- gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 91359105c..800b292da 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -27,7 +27,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 3) +version_info = (0, 5, 4) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 9c3eb3daf..d13e3aeb1 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 9c3eb3dafd765ee2e8299b53a1d8d780d9a8f55b +Subproject commit d13e3aeb168645965720ddec1f469e05771563ef diff --git a/setup.py b/setup.py index 7924b5a07..62bc6d007 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 3) +version_info = (0, 5, 4) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext':build_ext_nofail}, From e2b170a80462255dcc2003380db3547f55d8f14d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 8 Jul 2011 08:13:00 -0400 Subject: [PATCH 0196/1392] Workaround for #1 --- smmap/mman.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index ef9d43df0..f5b7efbd1 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -52,11 +52,14 @@ def _destroy(self): if self._rlist is not None: # Actual client count, which doesn't include the reference kept by the manager, nor ours # as we are about to be deleted - num_clients = self._rlist.client_count() - 2 - if num_clients == 0 and len(self._rlist) == 0: - # Free all resources associated with the mapped file - self._manager._fdict.pop(self._rlist.path_or_fd()) - #END remove regions list from manager + try: + num_clients = self._rlist.client_count() - 2 + if num_clients == 0 and len(self._rlist) == 0: + # Free all resources associated with the mapped file + self._manager._fdict.pop(self._rlist.path_or_fd()) + # END remove regions list from manager + except TypeError: + pass #END handle regions def _copy_from(self, rhs): From bdc1258abb48328a389720df2ffc404692add426 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 29 Aug 2011 21:45:59 +0200 Subject: [PATCH 0197/1392] Added LICENSE file containing a copy of (new)BSD --- LICENSE | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..710010f1f --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +Copyright (C) 2010, 2011 Sebastian Thiel and contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of the async project nor the names of +its contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + From 40fd4f31ab594dcfe049032c62ec61d2f0c3e492 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 18 Jan 2012 23:09:53 +0100 Subject: [PATCH 0198/1392] Added some more in-code comments to clarify why that exception is caught --- smmap/mman.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/smmap/mman.py b/smmap/mman.py index f5b7efbd1..7b0984358 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -59,7 +59,12 @@ def _destroy(self): self._manager._fdict.pop(self._rlist.path_or_fd()) # END remove regions list from manager except TypeError: + # sometimes, during shutdown, getrefcount is None. Its possible + # to re-import it, however, its probably better to just ignore + # this python problem (for now). + # The next step is to get rid of the error prone getrefcount alltogether. pass + #END exception handling #END handle regions def _copy_from(self, rhs): From 360a8956fe73a0a96315e946f52737569d990369 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 18 Jan 2012 23:12:02 +0100 Subject: [PATCH 0199/1392] Bumped version to 0.8.2 --- smmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index ae6e72eba..a10cd5c99 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 1) +version_info = (0, 8, 2) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From e96d2c381ef06667726eb745c67357de9d2a88fb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 23 Jul 2012 20:56:03 +0200 Subject: [PATCH 0200/1392] Submodules now use the http protocol to facilitate checkout in corporate networks --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 5dfd8e993..062ec5ba1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "async"] path = gitdb/ext/async - url = git://github.com/gitpython-developers/async.git + url = http://github.com/gitpython-developers/async.git [submodule "smmap"] path = gitdb/ext/smmap - url = git://github.com/Byron/smmap.git + url = http://github.com/Byron/smmap.git From ec6998f503e4619cd6bdecbbf372552ea126900a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 23 Jul 2012 21:12:03 +0200 Subject: [PATCH 0201/1392] Updated submodules to latest version --- gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/ext/async b/gitdb/ext/async index 10310824c..039c1d5c2 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 10310824c001deab8fea85b88ebda0696f964b3e +Subproject commit 039c1d5c26bc2ceaa9e55082efae2068d9873e45 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index d13e3aeb1..360a8956f 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit d13e3aeb168645965720ddec1f469e05771563ef +Subproject commit 360a8956fe73a0a96315e946f52737569d990369 From 0328caa516fffdbb5f28fd59798a9775aa2b05f5 Mon Sep 17 00:00:00 2001 From: David Black Date: Fri, 2 Nov 2012 10:43:48 +1100 Subject: [PATCH 0202/1392] Update gitmodules to point to the https location of the git repositories. Signed-off-by: David --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 062ec5ba1..978105388 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "async"] path = gitdb/ext/async - url = http://github.com/gitpython-developers/async.git + url = https://github.com/gitpython-developers/async.git [submodule "smmap"] path = gitdb/ext/smmap - url = http://github.com/Byron/smmap.git + url = https://github.com/Byron/smmap.git From 1b3ab5598e93369282502d049d64cb2ca12839cb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 9 Feb 2014 20:50:30 +0100 Subject: [PATCH 0203/1392] tabs to spaces --- setup.py | 0 smmap/buf.py | 246 ++++---- smmap/exc.py | 6 +- smmap/mman.py | 1122 +++++++++++++++++------------------ smmap/test/lib.py | 96 +-- smmap/test/test_buf.py | 204 +++---- smmap/test/test_mman.py | 400 ++++++------- smmap/test/test_tutorial.py | 160 ++--- smmap/test/test_util.py | 212 +++---- smmap/util.py | 464 +++++++-------- 10 files changed, 1455 insertions(+), 1455 deletions(-) mode change 100755 => 100644 setup.py diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 diff --git a/smmap/buf.py b/smmap/buf.py index 00ddbacd9..255c6b54d 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -6,129 +6,129 @@ __all__ = ["SlidingWindowMapBuffer"] class SlidingWindowMapBuffer(object): - """A buffer like object which allows direct byte-wise object and slicing into - memory of a mapped file. The mapping is controlled by the provided cursor. - - The buffer is relative, that is if you map an offset, index 0 will map to the - first byte at the offset you used during initialization or begin_access - - **Note:** Although this type effectively hides the fact that there are mapped windows - underneath, it can unfortunately not be used in any non-pure python method which - needs a buffer or string""" - __slots__ = ( - '_c', # our cursor - '_size', # our supposed size - ) - - - def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): - """Initalize the instance to operate on the given cursor. - :param cursor: if not None, the associated cursor to the file you want to access - If None, you have call begin_access before using the buffer and provide a cursor - :param offset: absolute offset in bytes - :param size: the total size of the mapping. Defaults to the maximum possible size - From that point on, the __len__ of the buffer will be the given size or the file size. - If the size is larger than the mappable area, you can only access the actually available - area, although the length of the buffer is reported to be your given size. - Hence it is in your own interest to provide a proper size ! - :param flags: Additional flags to be passed to os.open - :raise ValueError: if the buffer could not achieve a valid state""" - self._c = cursor - if cursor and not self.begin_access(cursor, offset, size, flags): - raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") - # END handle offset + """A buffer like object which allows direct byte-wise object and slicing into + memory of a mapped file. The mapping is controlled by the provided cursor. + + The buffer is relative, that is if you map an offset, index 0 will map to the + first byte at the offset you used during initialization or begin_access + + **Note:** Although this type effectively hides the fact that there are mapped windows + underneath, it can unfortunately not be used in any non-pure python method which + needs a buffer or string""" + __slots__ = ( + '_c', # our cursor + '_size', # our supposed size + ) + + + def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + """Initalize the instance to operate on the given cursor. + :param cursor: if not None, the associated cursor to the file you want to access + If None, you have call begin_access before using the buffer and provide a cursor + :param offset: absolute offset in bytes + :param size: the total size of the mapping. Defaults to the maximum possible size + From that point on, the __len__ of the buffer will be the given size or the file size. + If the size is larger than the mappable area, you can only access the actually available + area, although the length of the buffer is reported to be your given size. + Hence it is in your own interest to provide a proper size ! + :param flags: Additional flags to be passed to os.open + :raise ValueError: if the buffer could not achieve a valid state""" + self._c = cursor + if cursor and not self.begin_access(cursor, offset, size, flags): + raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds") + # END handle offset - def __del__(self): - self.end_access() - - def __len__(self): - return self._size - - def __getitem__(self, i): - c = self._c - assert c.is_valid() - if i < 0: - i = self._size + i - if not c.includes_ofs(i): - c.use_region(i, 1) - # END handle region usage - return c.buffer()[i-c.ofs_begin()] - - def __getslice__(self, i, j): - c = self._c - # fast path, slice fully included - safes a concatenate operation and - # should be the default - assert c.is_valid() - if i < 0: - i = self._size + i - if j == sys.maxint: - j = self._size - if j < 0: - j = self._size + j - if (c.ofs_begin() <= i) and (j < c.ofs_end()): - b = c.ofs_begin() - return c.buffer()[i-b:j-b] - else: - l = j-i # total length - ofs = i - # Keeping tokens in a list could possible be faster, but the list - # overhead outweighs the benefits (tested) ! - md = str() - while l: - c.use_region(ofs, l) - assert c.is_valid() - d = c.buffer()[:l] - ofs += len(d) - l -= len(d) - md += d - #END while there are bytes to read - return md - # END fast or slow path - #{ Interface - - def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): - """Call this before the first use of this instance. The method was already - called by the constructor in case sufficient information was provided. - - For more information no the parameters, see the __init__ method - :param path: if cursor is None the existing one will be used. - :return: True if the buffer can be used""" - if cursor: - self._c = cursor - #END update our cursor - - # reuse existing cursors if possible - if self._c is not None and self._c.is_associated(): - res = self._c.use_region(offset, size, flags).is_valid() - if res: - # if given size is too large or default, we computer a proper size - # If its smaller, we assume the combination between offset and size - # as chosen by the user is correct and use it ! - # If not, the user is in trouble. - if size > self._c.file_size(): - size = self._c.file_size() - offset - #END handle size - self._size = size - #END set size - return res - # END use our cursor - return False - - def end_access(self): - """Call this method once you are done using the instance. It is automatically - called on destruction, and should be called just in time to allow system - resources to be freed. - - Once you called end_access, you must call begin access before reusing this instance!""" - self._size = 0 - if self._c is not None: - self._c.unuse_region() - #END unuse region - - def cursor(self): - """:return: the currently set cursor which provides access to the data""" - return self._c - - #}END interface + def __del__(self): + self.end_access() + + def __len__(self): + return self._size + + def __getitem__(self, i): + c = self._c + assert c.is_valid() + if i < 0: + i = self._size + i + if not c.includes_ofs(i): + c.use_region(i, 1) + # END handle region usage + return c.buffer()[i-c.ofs_begin()] + + def __getslice__(self, i, j): + c = self._c + # fast path, slice fully included - safes a concatenate operation and + # should be the default + assert c.is_valid() + if i < 0: + i = self._size + i + if j == sys.maxint: + j = self._size + if j < 0: + j = self._size + j + if (c.ofs_begin() <= i) and (j < c.ofs_end()): + b = c.ofs_begin() + return c.buffer()[i-b:j-b] + else: + l = j-i # total length + ofs = i + # Keeping tokens in a list could possible be faster, but the list + # overhead outweighs the benefits (tested) ! + md = str() + while l: + c.use_region(ofs, l) + assert c.is_valid() + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + md += d + #END while there are bytes to read + return md + # END fast or slow path + #{ Interface + + def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + """Call this before the first use of this instance. The method was already + called by the constructor in case sufficient information was provided. + + For more information no the parameters, see the __init__ method + :param path: if cursor is None the existing one will be used. + :return: True if the buffer can be used""" + if cursor: + self._c = cursor + #END update our cursor + + # reuse existing cursors if possible + if self._c is not None and self._c.is_associated(): + res = self._c.use_region(offset, size, flags).is_valid() + if res: + # if given size is too large or default, we computer a proper size + # If its smaller, we assume the combination between offset and size + # as chosen by the user is correct and use it ! + # If not, the user is in trouble. + if size > self._c.file_size(): + size = self._c.file_size() - offset + #END handle size + self._size = size + #END set size + return res + # END use our cursor + return False + + def end_access(self): + """Call this method once you are done using the instance. It is automatically + called on destruction, and should be called just in time to allow system + resources to be freed. + + Once you called end_access, you must call begin access before reusing this instance!""" + self._size = 0 + if self._c is not None: + self._c.unuse_region() + #END unuse region + + def cursor(self): + """:return: the currently set cursor which provides access to the data""" + return self._c + + #}END interface diff --git a/smmap/exc.py b/smmap/exc.py index a090d24d5..f0ed7dcd8 100644 --- a/smmap/exc.py +++ b/smmap/exc.py @@ -1,7 +1,7 @@ """Module with system exceptions""" class MemoryManagerError(Exception): - """Base class for all exceptions thrown by the memory manager""" - + """Base class for all exceptions thrown by the memory manager""" + class RegionCollectionError(MemoryManagerError): - """Thrown if a memory region could not be collected, or if no region for collection was found""" + """Thrown if a memory region could not be collected, or if no region for collection was found""" diff --git a/smmap/mman.py b/smmap/mman.py index 7b0984358..97c42c5bb 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,11 +1,11 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" from util import ( - MapWindow, - MapRegion, - MapRegionList, - is_64_bit, - align_to_mmap - ) + MapWindow, + MapRegion, + MapRegionList, + is_64_bit, + align_to_mmap + ) from weakref import ref import sys @@ -18,564 +18,564 @@ class WindowCursor(object): - """ - Pointer into the mapped region of the memory manager, keeping the map - alive until it is destroyed and no other client uses it. + """ + Pointer into the mapped region of the memory manager, keeping the map + alive until it is destroyed and no other client uses it. - Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager - - **Note:**: The current implementation is suited for static and sliding window managers, but it also means - that it must be suited for the somewhat quite different sliding manager. It could be improved, but - I see no real need to do so.""" - __slots__ = ( - '_manager', # the manger keeping all file regions - '_rlist', # a regions list with regions for our file - '_region', # our current region or None - '_ofs', # relative offset from the actually mapped area to our start area - '_size' # maximum size we should provide - ) - - def __init__(self, manager = None, regions = None): - self._manager = manager - self._rlist = regions - self._region = None - self._ofs = 0 - self._size = 0 - - def __del__(self): - self._destroy() - - def _destroy(self): - """Destruction code to decrement counters""" - self.unuse_region() - - if self._rlist is not None: - # Actual client count, which doesn't include the reference kept by the manager, nor ours - # as we are about to be deleted - try: - num_clients = self._rlist.client_count() - 2 - if num_clients == 0 and len(self._rlist) == 0: - # Free all resources associated with the mapped file - self._manager._fdict.pop(self._rlist.path_or_fd()) - # END remove regions list from manager - except TypeError: - # sometimes, during shutdown, getrefcount is None. Its possible - # to re-import it, however, its probably better to just ignore - # this python problem (for now). - # The next step is to get rid of the error prone getrefcount alltogether. - pass - #END exception handling - #END handle regions - - def _copy_from(self, rhs): - """Copy all data from rhs into this instance, handles usage count""" - self._manager = rhs._manager - self._rlist = rhs._rlist - self._region = rhs._region - self._ofs = rhs._ofs - self._size = rhs._size - - if self._region is not None: - self._region.increment_usage_count() - # END handle regions - - def __copy__(self): - """copy module interface""" - cpy = type(self)() - cpy._copy_from(self) - return cpy - - #{ Interface - def assign(self, rhs): - """Assign rhs to this instance. This is required in order to get a real copy. - Alternativly, you can copy an existing instance using the copy module""" - self._destroy() - self._copy_from(rhs) - - def use_region(self, offset = 0, size = 0, flags = 0): - """Assure we point to a window which allows access to the given offset into the file - - :param offset: absolute offset in bytes into the file - :param size: amount of bytes to map. If 0, all available bytes will be mapped - :param flags: additional flags to be given to os.open in case a file handle is initially opened - for mapping. Has no effect if a region can actually be reused. - :return: this instance - it should be queried for whether it points to a valid memory region. - This is not the case if the mapping failed becaues we reached the end of the file - - **Note:**: The size actually mapped may be smaller than the given size. If that is the case, - either the file has reached its end, or the map was created between two existing regions""" - need_region = True - man = self._manager - fsize = self._rlist.file_size() - size = min(size or fsize, man.window_size() or fsize) # clamp size to window size - - if self._region is not None: - if self._region.includes_ofs(offset): - need_region = False - else: - self.unuse_region() - # END handle existing region - # END check existing region - - # offset too large ? - if offset >= fsize: - return self - #END handle offset - - if need_region: - self._region = man._obtain_region(self._rlist, offset, size, flags, False) - #END need region handling - - self._region.increment_usage_count() - self._ofs = offset - self._region._b - self._size = min(size, self._region.ofs_end() - offset) - - return self - - def unuse_region(self): - """Unuse the ucrrent region. Does nothing if we have no current region - - **Note:** the cursor unuses the region automatically upon destruction. It is recommended - to unuse the region once you are done reading from it in persistent cursors as it - helps to free up resource more quickly""" - self._region = None - # note: should reset ofs and size, but we spare that for performance. Its not - # allowed to query information if we are not valid ! + Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager + + **Note:**: The current implementation is suited for static and sliding window managers, but it also means + that it must be suited for the somewhat quite different sliding manager. It could be improved, but + I see no real need to do so.""" + __slots__ = ( + '_manager', # the manger keeping all file regions + '_rlist', # a regions list with regions for our file + '_region', # our current region or None + '_ofs', # relative offset from the actually mapped area to our start area + '_size' # maximum size we should provide + ) + + def __init__(self, manager = None, regions = None): + self._manager = manager + self._rlist = regions + self._region = None + self._ofs = 0 + self._size = 0 + + def __del__(self): + self._destroy() + + def _destroy(self): + """Destruction code to decrement counters""" + self.unuse_region() + + if self._rlist is not None: + # Actual client count, which doesn't include the reference kept by the manager, nor ours + # as we are about to be deleted + try: + num_clients = self._rlist.client_count() - 2 + if num_clients == 0 and len(self._rlist) == 0: + # Free all resources associated with the mapped file + self._manager._fdict.pop(self._rlist.path_or_fd()) + # END remove regions list from manager + except TypeError: + # sometimes, during shutdown, getrefcount is None. Its possible + # to re-import it, however, its probably better to just ignore + # this python problem (for now). + # The next step is to get rid of the error prone getrefcount alltogether. + pass + #END exception handling + #END handle regions + + def _copy_from(self, rhs): + """Copy all data from rhs into this instance, handles usage count""" + self._manager = rhs._manager + self._rlist = rhs._rlist + self._region = rhs._region + self._ofs = rhs._ofs + self._size = rhs._size + + if self._region is not None: + self._region.increment_usage_count() + # END handle regions + + def __copy__(self): + """copy module interface""" + cpy = type(self)() + cpy._copy_from(self) + return cpy + + #{ Interface + def assign(self, rhs): + """Assign rhs to this instance. This is required in order to get a real copy. + Alternativly, you can copy an existing instance using the copy module""" + self._destroy() + self._copy_from(rhs) + + def use_region(self, offset = 0, size = 0, flags = 0): + """Assure we point to a window which allows access to the given offset into the file + + :param offset: absolute offset in bytes into the file + :param size: amount of bytes to map. If 0, all available bytes will be mapped + :param flags: additional flags to be given to os.open in case a file handle is initially opened + for mapping. Has no effect if a region can actually be reused. + :return: this instance - it should be queried for whether it points to a valid memory region. + This is not the case if the mapping failed becaues we reached the end of the file + + **Note:**: The size actually mapped may be smaller than the given size. If that is the case, + either the file has reached its end, or the map was created between two existing regions""" + need_region = True + man = self._manager + fsize = self._rlist.file_size() + size = min(size or fsize, man.window_size() or fsize) # clamp size to window size + + if self._region is not None: + if self._region.includes_ofs(offset): + need_region = False + else: + self.unuse_region() + # END handle existing region + # END check existing region + + # offset too large ? + if offset >= fsize: + return self + #END handle offset + + if need_region: + self._region = man._obtain_region(self._rlist, offset, size, flags, False) + #END need region handling + + self._region.increment_usage_count() + self._ofs = offset - self._region._b + self._size = min(size, self._region.ofs_end() - offset) + + return self + + def unuse_region(self): + """Unuse the ucrrent region. Does nothing if we have no current region + + **Note:** the cursor unuses the region automatically upon destruction. It is recommended + to unuse the region once you are done reading from it in persistent cursors as it + helps to free up resource more quickly""" + self._region = None + # note: should reset ofs and size, but we spare that for performance. Its not + # allowed to query information if we are not valid ! - def buffer(self): - """Return a buffer object which allows access to our memory region from our offset - to the window size. Please note that it might be smaller than you requested when calling use_region() - - **Note:** You can only obtain a buffer if this instance is_valid() ! - - **Note:** buffers should not be cached passed the duration of your access as it will - prevent resources from being freed even though they might not be accounted for anymore !""" - return buffer(self._region.buffer(), self._ofs, self._size) - - def map(self): - """ - :return: the underlying raw memory map. Please not that the offset and size is likely to be different - to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole - file in case of StaticWindowMapManager""" - return self._region.map() - - def is_valid(self): - """:return: True if we have a valid and usable region""" - return self._region is not None - - def is_associated(self): - """:return: True if we are associated with a specific file already""" - return self._rlist is not None - - def ofs_begin(self): - """:return: offset to the first byte pointed to by our cursor - - **Note:** only if is_valid() is True""" - return self._region._b + self._ofs - - def ofs_end(self): - """:return: offset to one past the last available byte""" - # unroll method calls for performance ! - return self._region._b + self._ofs + self._size - - def size(self): - """:return: amount of bytes we point to""" - return self._size - - def region_ref(self): - """:return: weak ref to our mapped region. - :raise AssertionError: if we have no current region. This is only useful for debugging""" - if self._region is None: - raise AssertionError("region not set") - return ref(self._region) - - def includes_ofs(self, ofs): - """:return: True if the given absolute offset is contained in the cursors - current region - - **Note:** cursor must be valid for this to work""" - # unroll methods - return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) - - def file_size(self): - """:return: size of the underlying file""" - return self._rlist.file_size() - - def path_or_fd(self): - """:return: path or file decriptor of the underlying mapped file""" - return self._rlist.path_or_fd() + def buffer(self): + """Return a buffer object which allows access to our memory region from our offset + to the window size. Please note that it might be smaller than you requested when calling use_region() + + **Note:** You can only obtain a buffer if this instance is_valid() ! + + **Note:** buffers should not be cached passed the duration of your access as it will + prevent resources from being freed even though they might not be accounted for anymore !""" + return buffer(self._region.buffer(), self._ofs, self._size) + + def map(self): + """ + :return: the underlying raw memory map. Please not that the offset and size is likely to be different + to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole + file in case of StaticWindowMapManager""" + return self._region.map() + + def is_valid(self): + """:return: True if we have a valid and usable region""" + return self._region is not None + + def is_associated(self): + """:return: True if we are associated with a specific file already""" + return self._rlist is not None + + def ofs_begin(self): + """:return: offset to the first byte pointed to by our cursor + + **Note:** only if is_valid() is True""" + return self._region._b + self._ofs + + def ofs_end(self): + """:return: offset to one past the last available byte""" + # unroll method calls for performance ! + return self._region._b + self._ofs + self._size + + def size(self): + """:return: amount of bytes we point to""" + return self._size + + def region_ref(self): + """:return: weak ref to our mapped region. + :raise AssertionError: if we have no current region. This is only useful for debugging""" + if self._region is None: + raise AssertionError("region not set") + return ref(self._region) + + def includes_ofs(self, ofs): + """:return: True if the given absolute offset is contained in the cursors + current region + + **Note:** cursor must be valid for this to work""" + # unroll methods + return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) + + def file_size(self): + """:return: size of the underlying file""" + return self._rlist.file_size() + + def path_or_fd(self): + """:return: path or file decriptor of the underlying mapped file""" + return self._rlist.path_or_fd() - def path(self): - """:return: path of the underlying mapped file - :raise ValueError: if attached path is not a path""" - if isinstance(self._rlist.path_or_fd(), int): - raise ValueError("Path queried although mapping was applied to a file descriptor") - # END handle type - return self._rlist.path_or_fd() - - def fd(self): - """:return: file descriptor used to create the underlying mapping. - - **Note:** it is not required to be valid anymore - :raise ValueError: if the mapping was not created by a file descriptor""" - if isinstance(self._rlist.path_or_fd(), basestring): - raise ValueError("File descriptor queried although mapping was generated from path") - #END handle type - return self._rlist.path_or_fd() - - #} END interface - - + def path(self): + """:return: path of the underlying mapped file + :raise ValueError: if attached path is not a path""" + if isinstance(self._rlist.path_or_fd(), int): + raise ValueError("Path queried although mapping was applied to a file descriptor") + # END handle type + return self._rlist.path_or_fd() + + def fd(self): + """:return: file descriptor used to create the underlying mapping. + + **Note:** it is not required to be valid anymore + :raise ValueError: if the mapping was not created by a file descriptor""" + if isinstance(self._rlist.path_or_fd(), basestring): + raise ValueError("File descriptor queried although mapping was generated from path") + #END handle type + return self._rlist.path_or_fd() + + #} END interface + + class StaticWindowMapManager(object): - """Provides a manager which will produce single size cursors that are allowed - to always map the whole file. - - Clients must be written to specifically know that they are accessing their data - through a StaticWindowMapManager, as they otherwise have to deal with their window size. - - These clients would have to use a SlidingWindowMapBuffer to hide this fact. - - This type will always use a maximum window size, and optimize certain methods to - acomodate this fact""" - - __slots__ = [ - '_fdict', # mapping of path -> StorageHelper (of some kind - '_window_size', # maximum size of a window - '_max_memory_size', # maximum amount ofmemory we may allocate - '_max_handle_count', # maximum amount of handles to keep open - '_memory_size', # currently allocated memory size - '_handle_count', # amount of currently allocated file handles - ] - - #{ Configuration - MapRegionListCls = MapRegionList - MapWindowCls = MapWindow - MapRegionCls = MapRegion - WindowCursorCls = WindowCursor - #} END configuration - - _MB_in_bytes = 1024 * 1024 - - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): - """initialize the manager with the given parameters. - :param window_size: if -1, a default window size will be chosen depending on - the operating system's architechture. It will internally be quantified to a multiple of the page size - If 0, the window may have any size, which basically results in mapping the whole file at one - :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. - If 0, a viable default iwll be set dependning on the system's architecture. - It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate - :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. - Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, - the manager will free as many handles as posisble""" - self._fdict = dict() - self._window_size = window_size - self._max_memory_size = max_memory_size - self._max_handle_count = max_open_handles - self._memory_size = 0 - self._handle_count = 0 - - if window_size < 0: - coeff = 32 - if is_64_bit(): - coeff = 1024 - #END handle arch - self._window_size = coeff * self._MB_in_bytes - # END handle max window size - - if max_memory_size == 0: - coeff = 512 - if is_64_bit(): - coeff = 8192 - #END handle arch - self._max_memory_size = coeff * self._MB_in_bytes - #END handle max memory size - - #{ Internal Methods - - def _collect_lru_region(self, size): - """Unmap the region which was least-recently used and has no client - :param size: size of the region we want to map next (assuming its not already mapped partially or full - if 0, we try to free any available region - :return: Amount of freed regions - - **Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. - If the system runs out of memory, it will tell. - - **todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" - num_found = 0 - while (size == 0) or (self._memory_size + size > self._max_memory_size): - lru_region = None - lru_list = None - for regions in self._fdict.itervalues(): - for region in regions: - # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-2 == 0 and - (lru_region is None or region._uc < lru_region._uc)): - lru_region = region - lru_list = regions - # END update lru_region - #END for each region - #END for each regions list - - if lru_region is None: - break - #END handle region not found - - num_found += 1 - del(lru_list[lru_list.index(lru_region)]) - self._memory_size -= lru_region.size() - self._handle_count -= 1 - #END while there is more memory to free - return num_found - - def _obtain_region(self, a, offset, size, flags, is_recursive): - """Utilty to create a new region - for more information on the parameters, - see MapCursor.use_region. - :param a: A regions (a)rray - :return: The newly created region""" - if self._memory_size + size > self._max_memory_size: - self._collect_lru_region(size) - #END handle collection - - r = None - if a: - assert len(a) == 1 - r = a[0] - else: - try: - r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxint, flags) - except Exception: - # apparently we are out of system resources or hit a limit - # As many more operations are likely to fail in that condition ( - # like reading a file from disk, etc) we free up as much as possible - # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway - if is_recursive: - # we already tried this, and still have no success in obtaining - # a mapping. This is an exception, so we propagate it - raise - #END handle existing recursion - self._collect_lru_region(0) - return self._obtain_region(a, offset, size, flags, True) - #END handle exceptions - - self._handle_count += 1 - self._memory_size += r.size() - a.append(r) - # END handle array - - assert r.includes_ofs(offset) - return r + """Provides a manager which will produce single size cursors that are allowed + to always map the whole file. + + Clients must be written to specifically know that they are accessing their data + through a StaticWindowMapManager, as they otherwise have to deal with their window size. + + These clients would have to use a SlidingWindowMapBuffer to hide this fact. + + This type will always use a maximum window size, and optimize certain methods to + acomodate this fact""" + + __slots__ = [ + '_fdict', # mapping of path -> StorageHelper (of some kind + '_window_size', # maximum size of a window + '_max_memory_size', # maximum amount ofmemory we may allocate + '_max_handle_count', # maximum amount of handles to keep open + '_memory_size', # currently allocated memory size + '_handle_count', # amount of currently allocated file handles + ] + + #{ Configuration + MapRegionListCls = MapRegionList + MapWindowCls = MapWindow + MapRegionCls = MapRegion + WindowCursorCls = WindowCursor + #} END configuration + + _MB_in_bytes = 1024 * 1024 + + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): + """initialize the manager with the given parameters. + :param window_size: if -1, a default window size will be chosen depending on + the operating system's architechture. It will internally be quantified to a multiple of the page size + If 0, the window may have any size, which basically results in mapping the whole file at one + :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. + If 0, a viable default iwll be set dependning on the system's architecture. + It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate + :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. + Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, + the manager will free as many handles as posisble""" + self._fdict = dict() + self._window_size = window_size + self._max_memory_size = max_memory_size + self._max_handle_count = max_open_handles + self._memory_size = 0 + self._handle_count = 0 + + if window_size < 0: + coeff = 32 + if is_64_bit(): + coeff = 1024 + #END handle arch + self._window_size = coeff * self._MB_in_bytes + # END handle max window size + + if max_memory_size == 0: + coeff = 512 + if is_64_bit(): + coeff = 8192 + #END handle arch + self._max_memory_size = coeff * self._MB_in_bytes + #END handle max memory size + + #{ Internal Methods + + def _collect_lru_region(self, size): + """Unmap the region which was least-recently used and has no client + :param size: size of the region we want to map next (assuming its not already mapped partially or full + if 0, we try to free any available region + :return: Amount of freed regions + + **Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. + If the system runs out of memory, it will tell. + + **todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + num_found = 0 + while (size == 0) or (self._memory_size + size > self._max_memory_size): + lru_region = None + lru_list = None + for regions in self._fdict.itervalues(): + for region in regions: + # check client count - consider that we keep one reference ourselves ! + if (region.client_count()-2 == 0 and + (lru_region is None or region._uc < lru_region._uc)): + lru_region = region + lru_list = regions + # END update lru_region + #END for each region + #END for each regions list + + if lru_region is None: + break + #END handle region not found + + num_found += 1 + del(lru_list[lru_list.index(lru_region)]) + self._memory_size -= lru_region.size() + self._handle_count -= 1 + #END while there is more memory to free + return num_found + + def _obtain_region(self, a, offset, size, flags, is_recursive): + """Utilty to create a new region - for more information on the parameters, + see MapCursor.use_region. + :param a: A regions (a)rray + :return: The newly created region""" + if self._memory_size + size > self._max_memory_size: + self._collect_lru_region(size) + #END handle collection + + r = None + if a: + assert len(a) == 1 + r = a[0] + else: + try: + r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxint, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + #END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + a.append(r) + # END handle array + + assert r.includes_ofs(offset) + return r - #}END internal methods - - #{ Interface - def make_cursor(self, path_or_fd): - """ - :return: a cursor pointing to the given path or file descriptor. - It can be used to map new regions of the file into memory - - **Note:** if a file descriptor is given, it is assumed to be open and valid, - but may be closed afterwards. To refer to the same file, you may reuse - your existing file descriptor, but keep in mind that new windows can only - be mapped as long as it stays valid. This is why the using actual file paths - are preferred unless you plan to keep the file descriptor open. - - **Note:** file descriptors are problematic as they are not necessarily unique, as two - different files opened and closed in succession might have the same file descriptor id. - - **Note:** Using file descriptors directly is faster once new windows are mapped as it - prevents the file to be opened again just for the purpose of mapping it.""" - regions = self._fdict.get(path_or_fd) - if regions is None: - regions = self.MapRegionListCls(path_or_fd) - self._fdict[path_or_fd] = regions - # END obtain region for path - return self.WindowCursorCls(self, regions) - - def collect(self): - """Collect all available free-to-collect mapped regions - :return: Amount of freed handles""" - return self._collect_lru_region(0) - - def num_file_handles(self): - """:return: amount of file handles in use. Each mapped region uses one file handle""" - return self._handle_count - - def num_open_files(self): - """Amount of opened files in the system""" - return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) - - def window_size(self): - """:return: size of each window when allocating new regions""" - return self._window_size - - def mapped_memory_size(self): - """:return: amount of bytes currently mapped in total""" - return self._memory_size - - def max_file_handles(self): - """:return: maximium amount of handles we may have opened""" - return self._max_handle_count - - def max_mapped_memory_size(self): - """:return: maximum amount of memory we may allocate""" - return self._max_memory_size - - #} END interface - - #{ Special Purpose Interface - - def force_map_handle_removal_win(self, base_path): - """ONLY AVAILABLE ON WINDOWS - On windows removing files is not allowed if anybody still has it opened. - If this process is ourselves, and if the whole process uses this memory - manager (as far as the parent framework is concerned) we can enforce - closing all memory maps whose path matches the given base path to - allow the respective operation after all. - The respective system must NOT access the closed memory regions anymore ! - This really may only be used if you know that the items which keep - the cursors alive will not be using it anymore. They need to be recreated ! - :return: Amount of closed handles - - **Note:** does nothing on non-windows platforms""" - if sys.platform != 'win32': - return - #END early bailout - - num_closed = 0 - for path, rlist in self._fdict.iteritems(): - if path.startswith(base_path): - for region in rlist: - region._mf.close() - num_closed += 1 - #END path matches - #END for each path - return num_closed - #} END special purpose interface - - - + #}END internal methods + + #{ Interface + def make_cursor(self, path_or_fd): + """ + :return: a cursor pointing to the given path or file descriptor. + It can be used to map new regions of the file into memory + + **Note:** if a file descriptor is given, it is assumed to be open and valid, + but may be closed afterwards. To refer to the same file, you may reuse + your existing file descriptor, but keep in mind that new windows can only + be mapped as long as it stays valid. This is why the using actual file paths + are preferred unless you plan to keep the file descriptor open. + + **Note:** file descriptors are problematic as they are not necessarily unique, as two + different files opened and closed in succession might have the same file descriptor id. + + **Note:** Using file descriptors directly is faster once new windows are mapped as it + prevents the file to be opened again just for the purpose of mapping it.""" + regions = self._fdict.get(path_or_fd) + if regions is None: + regions = self.MapRegionListCls(path_or_fd) + self._fdict[path_or_fd] = regions + # END obtain region for path + return self.WindowCursorCls(self, regions) + + def collect(self): + """Collect all available free-to-collect mapped regions + :return: Amount of freed handles""" + return self._collect_lru_region(0) + + def num_file_handles(self): + """:return: amount of file handles in use. Each mapped region uses one file handle""" + return self._handle_count + + def num_open_files(self): + """Amount of opened files in the system""" + return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) + + def window_size(self): + """:return: size of each window when allocating new regions""" + return self._window_size + + def mapped_memory_size(self): + """:return: amount of bytes currently mapped in total""" + return self._memory_size + + def max_file_handles(self): + """:return: maximium amount of handles we may have opened""" + return self._max_handle_count + + def max_mapped_memory_size(self): + """:return: maximum amount of memory we may allocate""" + return self._max_memory_size + + #} END interface + + #{ Special Purpose Interface + + def force_map_handle_removal_win(self, base_path): + """ONLY AVAILABLE ON WINDOWS + On windows removing files is not allowed if anybody still has it opened. + If this process is ourselves, and if the whole process uses this memory + manager (as far as the parent framework is concerned) we can enforce + closing all memory maps whose path matches the given base path to + allow the respective operation after all. + The respective system must NOT access the closed memory regions anymore ! + This really may only be used if you know that the items which keep + the cursors alive will not be using it anymore. They need to be recreated ! + :return: Amount of closed handles + + **Note:** does nothing on non-windows platforms""" + if sys.platform != 'win32': + return + #END early bailout + + num_closed = 0 + for path, rlist in self._fdict.iteritems(): + if path.startswith(base_path): + for region in rlist: + region._mf.close() + num_closed += 1 + #END path matches + #END for each path + return num_closed + #} END special purpose interface + + + class SlidingWindowMapManager(StaticWindowMapManager): - """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily - obtain additional regions assuring there is no overlap. - Once a certain memory limit is reached globally, or if there cannot be more open file handles - which result from each mmap call, the least recently used, and currently unused mapped regions - are unloaded automatically. - - **Note:** currently not thread-safe ! - - **Note:** in the current implementation, we will automatically unload windows if we either cannot - create more memory maps (as the open file handles limit is hit) or if we have allocated more than - a safe amount of memory already, which would possibly cause memory allocations to fail as our address - space is full.""" - - __slots__ = tuple() - - def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxint): - """Adjusts the default window size to -1""" - super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) - - def _obtain_region(self, a, offset, size, flags, is_recursive): - # bisect to find an existing region. The c++ implementation cannot - # do that as it uses a linked list for regions. - r = None - lo = 0 - hi = len(a) - while lo < hi: - mid = (lo+hi)//2 - ofs = a[mid]._b - if ofs <= offset: - if a[mid].includes_ofs(offset): - r = a[mid] - break - #END have region - lo = mid+1 - else: - hi = mid - #END handle position - #END while bisecting - - if r is None: - window_size = self._window_size - left = self.MapWindowCls(0, 0) - mid = self.MapWindowCls(offset, size) - right = self.MapWindowCls(a.file_size(), 0) - - # we want to honor the max memory size, and assure we have anough - # memory available - # Save calls ! - if self._memory_size + window_size > self._max_memory_size: - self._collect_lru_region(window_size) - #END handle collection - - # we assume the list remains sorted by offset - insert_pos = 0 - len_regions = len(a) - if len_regions == 1: - if a[0]._b <= offset: - insert_pos = 1 - #END maintain sort - else: - # find insert position - insert_pos = len_regions - for i, region in enumerate(a): - if region._b > offset: - insert_pos = i - break - #END if insert position is correct - #END for each region - # END obtain insert pos - - # adjust the actual offset and size values to create the largest - # possible mapping - if insert_pos == 0: - if len_regions: - right = self.MapWindowCls.from_region(a[insert_pos]) - #END adjust right side - else: - if insert_pos != len_regions: - right = self.MapWindowCls.from_region(a[insert_pos]) - # END adjust right window - left = self.MapWindowCls.from_region(a[insert_pos - 1]) - #END adjust surrounding windows - - mid.extend_left_to(left, window_size) - mid.extend_right_to(right, window_size) - mid.align() - - # it can happen that we align beyond the end of the file - if mid.ofs_end() > right.ofs: - mid.size = right.ofs - mid.ofs - #END readjust size - - # insert new region at the right offset to keep the order - try: - if self._handle_count >= self._max_handle_count: - raise Exception - #END assert own imposed max file handles - r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) - except Exception: - # apparently we are out of system resources or hit a limit - # As many more operations are likely to fail in that condition ( - # like reading a file from disk, etc) we free up as much as possible - # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway - if is_recursive: - # we already tried this, and still have no success in obtaining - # a mapping. This is an exception, so we propagate it - raise - #END handle existing recursion - self._collect_lru_region(0) - return self._obtain_region(a, offset, size, flags, True) - #END handle exceptions - - self._handle_count += 1 - self._memory_size += r.size() - a.insert(insert_pos, r) - # END create new region - return r - - + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily + obtain additional regions assuring there is no overlap. + Once a certain memory limit is reached globally, or if there cannot be more open file handles + which result from each mmap call, the least recently used, and currently unused mapped regions + are unloaded automatically. + + **Note:** currently not thread-safe ! + + **Note:** in the current implementation, we will automatically unload windows if we either cannot + create more memory maps (as the open file handles limit is hit) or if we have allocated more than + a safe amount of memory already, which would possibly cause memory allocations to fail as our address + space is full.""" + + __slots__ = tuple() + + def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxint): + """Adjusts the default window size to -1""" + super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) + + def _obtain_region(self, a, offset, size, flags, is_recursive): + # bisect to find an existing region. The c++ implementation cannot + # do that as it uses a linked list for regions. + r = None + lo = 0 + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + ofs = a[mid]._b + if ofs <= offset: + if a[mid].includes_ofs(offset): + r = a[mid] + break + #END have region + lo = mid+1 + else: + hi = mid + #END handle position + #END while bisecting + + if r is None: + window_size = self._window_size + left = self.MapWindowCls(0, 0) + mid = self.MapWindowCls(offset, size) + right = self.MapWindowCls(a.file_size(), 0) + + # we want to honor the max memory size, and assure we have anough + # memory available + # Save calls ! + if self._memory_size + window_size > self._max_memory_size: + self._collect_lru_region(window_size) + #END handle collection + + # we assume the list remains sorted by offset + insert_pos = 0 + len_regions = len(a) + if len_regions == 1: + if a[0]._b <= offset: + insert_pos = 1 + #END maintain sort + else: + # find insert position + insert_pos = len_regions + for i, region in enumerate(a): + if region._b > offset: + insert_pos = i + break + #END if insert position is correct + #END for each region + # END obtain insert pos + + # adjust the actual offset and size values to create the largest + # possible mapping + if insert_pos == 0: + if len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + #END adjust right side + else: + if insert_pos != len_regions: + right = self.MapWindowCls.from_region(a[insert_pos]) + # END adjust right window + left = self.MapWindowCls.from_region(a[insert_pos - 1]) + #END adjust surrounding windows + + mid.extend_left_to(left, window_size) + mid.extend_right_to(right, window_size) + mid.align() + + # it can happen that we align beyond the end of the file + if mid.ofs_end() > right.ofs: + mid.size = right.ofs - mid.ofs + #END readjust size + + # insert new region at the right offset to keep the order + try: + if self._handle_count >= self._max_handle_count: + raise Exception + #END assert own imposed max file handles + r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) + except Exception: + # apparently we are out of system resources or hit a limit + # As many more operations are likely to fail in that condition ( + # like reading a file from disk, etc) we free up as much as possible + # As this invalidates our insert position, we have to recurse here + # NOTE: The c++ version uses a linked list to curcumvent this, but + # using that in python is probably too slow anyway + if is_recursive: + # we already tried this, and still have no success in obtaining + # a mapping. This is an exception, so we propagate it + raise + #END handle existing recursion + self._collect_lru_region(0) + return self._obtain_region(a, offset, size, flags, True) + #END handle exceptions + + self._handle_count += 1 + self._memory_size += r.size() + a.insert(insert_pos, r) + # END create new region + return r + + diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 6957dcab0..21e6c5a09 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -9,57 +9,57 @@ #{ Utilities class FileCreator(object): - """A instance which creates a temporary file with a prefix and a given size - and provides this info to the user. - Once it gets deleted, it will remove the temporary file as well.""" - __slots__ = ("_size", "_path") - - def __init__(self, size, prefix=''): - assert size, "Require size to be larger 0" - - self._path = tempfile.mktemp(prefix=prefix) - self._size = size - - fp = open(self._path, "wb") - fp.seek(size-1) - fp.write('1') - fp.close() - - assert os.path.getsize(self.path) == size + """A instance which creates a temporary file with a prefix and a given size + and provides this info to the user. + Once it gets deleted, it will remove the temporary file as well.""" + __slots__ = ("_size", "_path") + + def __init__(self, size, prefix=''): + assert size, "Require size to be larger 0" + + self._path = tempfile.mktemp(prefix=prefix) + self._size = size + + fp = open(self._path, "wb") + fp.seek(size-1) + fp.write('1') + fp.close() + + assert os.path.getsize(self.path) == size - def __del__(self): - try: - os.remove(self.path) - except OSError: - pass - #END exception handling - + def __del__(self): + try: + os.remove(self.path) + except OSError: + pass + #END exception handling + - @property - def path(self): - return self._path - - @property - def size(self): - return self._size + @property + def path(self): + return self._path + + @property + def size(self): + return self._size #} END utilities class TestBase(TestCase): - """Foundation used by all tests""" - - #{ Configuration - k_window_test_size = 1000 * 1000 * 8 + 5195 - #} END configuration - - #{ Overrides - @classmethod - def setUpAll(cls): - # nothing for now - pass - - #END overrides - - #{ Interface - - #} END interface + """Foundation used by all tests""" + + #{ Configuration + k_window_test_size = 1000 * 1000 * 8 + 5195 + #} END configuration + + #{ Overrides + @classmethod + def setUpAll(cls): + # nothing for now + pass + + #END overrides + + #{ Interface + + #} END interface diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 9881c6294..4bdcb76f5 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -10,108 +10,108 @@ man_optimal = SlidingWindowMapManager() -man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, - max_memory_size=TestBase.k_window_test_size/3, - max_open_handles=15) +man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, + max_memory_size=TestBase.k_window_test_size/3, + max_open_handles=15) static_man = StaticWindowMapManager() class TestBuf(TestBase): - - def test_basics(self): - fc = FileCreator(self.k_window_test_size, "buffer_test") - - # invalid paths fail upon construction - c = man_optimal.make_cursor(fc.path) - self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor - self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large - - buf = SlidingWindowMapBuffer() # can create uninitailized buffers - assert buf.cursor() is None - - # can call end access any time - buf.end_access() - buf.end_access() - assert len(buf) == 0 - - # begin access can revive it, if the offset is suitable - offset = 100 - assert buf.begin_access(c, fc.size) == False - assert buf.begin_access(c, offset) == True - assert len(buf) == fc.size - offset - assert buf.cursor().is_valid() - - # empty begin access keeps it valid on the same path, but alters the offset - assert buf.begin_access() == True - assert len(buf) == fc.size - assert buf.cursor().is_valid() - - # simple access - data = open(fc.path, 'rb').read() - assert data[offset] == buf[0] - assert data[offset:offset*2] == buf[0:offset] - - # negative indices, partial slices - assert buf[-1] == buf[len(buf)-1] - assert buf[-10:] == buf[len(buf)-10:len(buf)] - - # end access makes its cursor invalid - buf.end_access() - assert not buf.cursor().is_valid() - assert buf.cursor().is_associated() # but it remains associated - - # an empty begin access fixes it up again - assert buf.begin_access() == True and buf.cursor().is_valid() - del(buf) # ends access automatically - del(c) - - assert man_optimal.num_file_handles() == 1 - - # PERFORMANCE - # blast away with rnadom access and a full mapping - we don't want to - # exagerate the manager's overhead, but measure the buffer overhead - # We do it once with an optimal setting, and with a worse manager which - # will produce small mappings only ! - max_num_accesses = 100 - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - for manager, man_id in ( (man_optimal, 'optimal'), - (man_worst_case, 'worst case'), - (static_man, 'static optimial')): - buf = SlidingWindowMapBuffer(manager.make_cursor(item)) - assert manager.num_file_handles() == 1 - for access_mode in range(2): # single, multi - num_accesses_left = max_num_accesses - num_bytes = 0 - fsize = fc.size - - st = time() - buf.begin_access() - while num_accesses_left: - num_accesses_left -= 1 - if access_mode: # multi - ofs_start = randint(0, fsize) - ofs_end = randint(ofs_start, fsize) - d = buf[ofs_start:ofs_end] - assert len(d) == ofs_end - ofs_start - assert d == data[ofs_start:ofs_end] - num_bytes += len(d) - else: - pos = randint(0, fsize) - assert buf[pos] == data[pos] - num_bytes += 1 - #END handle mode - # END handle num accesses - - buf.end_access() - assert manager.num_file_handles() - assert manager.collect() - assert manager.num_file_handles() == 0 - elapsed = max(time() - st, 0.001) # prevent zero division errors on windows - mb = float(1000*1000) - mode_str = (access_mode and "slice") or "single byte" - sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" - % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) - # END handle access mode - # END for each manager - # END for each input - os.close(fd) + + def test_basics(self): + fc = FileCreator(self.k_window_test_size, "buffer_test") + + # invalid paths fail upon construction + c = man_optimal.make_cursor(fc.path) + self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor + self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large + + buf = SlidingWindowMapBuffer() # can create uninitailized buffers + assert buf.cursor() is None + + # can call end access any time + buf.end_access() + buf.end_access() + assert len(buf) == 0 + + # begin access can revive it, if the offset is suitable + offset = 100 + assert buf.begin_access(c, fc.size) == False + assert buf.begin_access(c, offset) == True + assert len(buf) == fc.size - offset + assert buf.cursor().is_valid() + + # empty begin access keeps it valid on the same path, but alters the offset + assert buf.begin_access() == True + assert len(buf) == fc.size + assert buf.cursor().is_valid() + + # simple access + data = open(fc.path, 'rb').read() + assert data[offset] == buf[0] + assert data[offset:offset*2] == buf[0:offset] + + # negative indices, partial slices + assert buf[-1] == buf[len(buf)-1] + assert buf[-10:] == buf[len(buf)-10:len(buf)] + + # end access makes its cursor invalid + buf.end_access() + assert not buf.cursor().is_valid() + assert buf.cursor().is_associated() # but it remains associated + + # an empty begin access fixes it up again + assert buf.begin_access() == True and buf.cursor().is_valid() + del(buf) # ends access automatically + del(c) + + assert man_optimal.num_file_handles() == 1 + + # PERFORMANCE + # blast away with rnadom access and a full mapping - we don't want to + # exagerate the manager's overhead, but measure the buffer overhead + # We do it once with an optimal setting, and with a worse manager which + # will produce small mappings only ! + max_num_accesses = 100 + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + for manager, man_id in ( (man_optimal, 'optimal'), + (man_worst_case, 'worst case'), + (static_man, 'static optimial')): + buf = SlidingWindowMapBuffer(manager.make_cursor(item)) + assert manager.num_file_handles() == 1 + for access_mode in range(2): # single, multi + num_accesses_left = max_num_accesses + num_bytes = 0 + fsize = fc.size + + st = time() + buf.begin_access() + while num_accesses_left: + num_accesses_left -= 1 + if access_mode: # multi + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + else: + pos = randint(0, fsize) + assert buf[pos] == data[pos] + num_bytes += 1 + #END handle mode + # END handle num accesses + + buf.end_access() + assert manager.num_file_handles() + assert manager.collect() + assert manager.num_file_handles() == 0 + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows + mb = float(1000*1000) + mode_str = (access_mode and "slice") or "single byte" + sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" + % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + # END handle access mode + # END for each manager + # END for each input + os.close(fd) diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 27be686ad..46429a419 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -12,203 +12,203 @@ from copy import copy class TestMMan(TestBase): - - def test_cursor(self): - fc = FileCreator(self.k_window_test_size, "cursor_test") - - man = SlidingWindowMapManager() - ci = WindowCursor(man) # invalid cursor - assert not ci.is_valid() - assert not ci.is_associated() - assert ci.size() == 0 # this is cached, so we can query it in invalid state - - cv = man.make_cursor(fc.path) - assert not cv.is_valid() # no region mapped yet - assert cv.is_associated()# but it know where to map it from - assert cv.file_size() == fc.size - assert cv.path() == fc.path - - # copy module - cio = copy(cv) - assert not cio.is_valid() and cio.is_associated() - - # assign method - assert not ci.is_associated() - ci.assign(cv) - assert not ci.is_valid() and ci.is_associated() - - # unuse non-existing region is fine - cv.unuse_region() - cv.unuse_region() - - # destruction is fine (even multiple times) - cv._destroy() - WindowCursor(man)._destroy() - - def test_memory_manager(self): - slide_man = SlidingWindowMapManager() - static_man = StaticWindowMapManager() - - for man in (static_man, slide_man): - assert man.num_file_handles() == 0 - assert man.num_open_files() == 0 - winsize_cmp_val = 0 - if isinstance(man, StaticWindowMapManager): - winsize_cmp_val = -1 - #END handle window size - assert man.window_size() > winsize_cmp_val - assert man.mapped_memory_size() == 0 - assert man.max_mapped_memory_size() > 0 - - # collection doesn't raise in 'any' mode - man._collect_lru_region(0) - # doesn't raise if we are within the limit - man._collect_lru_region(10) - - # doesn't fail if we overallocate - assert man._collect_lru_region(sys.maxint) == 0 - - # use a region, verify most basic functionality - fc = FileCreator(self.k_window_test_size, "manager_test") - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - c = man.make_cursor(item) - assert c.path_or_fd() is item - assert c.use_region(10, 10).is_valid() - assert c.ofs_begin() == 10 - assert c.size() == 10 - assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] - - if isinstance(item, int): - self.failUnlessRaises(ValueError, c.path) - else: - self.failUnlessRaises(ValueError, c.fd) - #END handle value error - #END for each input - os.close(fd) - # END for each manager type - - def test_memman_operation(self): - # test more access, force it to actually unmap regions - fc = FileCreator(self.k_window_test_size, "manager_operation_test") - data = open(fc.path, 'rb').read() - fd = os.open(fc.path, os.O_RDONLY) - max_num_handles = 15 - #small_size = - for mtype, args in ( (StaticWindowMapManager, (0, fc.size / 3, max_num_handles)), - (SlidingWindowMapManager, (fc.size / 100, fc.size / 3, max_num_handles)),): - for item in (fc.path, fd): - assert len(data) == fc.size - - # small windows, a reasonable max memory. Not too many regions at once - man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) - c = man.make_cursor(item) - - # still empty (more about that is tested in test_memory_manager() - assert man.num_open_files() == 0 - assert man.mapped_memory_size() == 0 - - base_offset = 5000 - # window size is 0 for static managers, hence size will be 0. We take that into consideration - size = man.window_size() / 2 - assert c.use_region(base_offset, size).is_valid() - rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us - - assert man.num_open_files() == 1 - assert man.num_file_handles() == 1 - assert man.mapped_memory_size() == rr().size() - - #assert c.size() == size # the cursor may overallocate in its static version - assert c.ofs_begin() == base_offset - assert rr().ofs_begin() == 0 # it was aligned and expanded - if man.window_size(): - assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) - else: - assert rr().size() == fc.size - #END ignore static managers which dont use windows and are aligned to file boundaries - - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] - - # obtain second window, which spans the first part of the file - it is a still the same window - nsize = (size or fc.size) - 10 - assert c.use_region(0, nsize).is_valid() - assert c.region_ref()() == rr() - assert man.num_file_handles() == 1 - assert c.size() == nsize - assert c.ofs_begin() == 0 - assert c.buffer()[:] == data[:nsize] - - # map some part at the end, our requested size cannot be kept - overshoot = 4000 - base_offset = fc.size - (size or c.size()) + overshoot - assert c.use_region(base_offset, size).is_valid() - if man.window_size(): - assert man.num_file_handles() == 2 - assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager - else: - assert c.size() < fc.size - #END ignore static managers which only have one handle per file - rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] - - # unising a region makes the cursor invalid - c.unuse_region() - assert not c.is_valid() - if man.window_size(): - # but doesn't change anything regarding the handle count - we cache it and only - # remove mapped regions if we have to - assert man.num_file_handles() == 2 - #END ignore this for static managers - - # iterate through the windows, verify data contents - # this will trigger map collection after a while - max_random_accesses = 5000 - num_random_accesses = max_random_accesses - memory_read = 0 - st = time() - - # cache everything to get some more performance - includes_ofs = c.includes_ofs - max_mapped_memory_size = man.max_mapped_memory_size() - max_file_handles = man.max_file_handles() - mapped_memory_size = man.mapped_memory_size - num_file_handles = man.num_file_handles - while num_random_accesses: - num_random_accesses -= 1 - base_offset = randint(0, fc.size - 1) - - # precondition - if man.window_size(): - assert max_mapped_memory_size >= mapped_memory_size() - #END statics will overshoot, which is fine - assert max_file_handles >= num_file_handles() - assert c.use_region(base_offset, (size or c.size())).is_valid() - csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset+csize] - memory_read += csize - - assert includes_ofs(base_offset) - assert includes_ofs(base_offset+csize-1) - assert not includes_ofs(base_offset+csize) - # END while we should do an access - elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows - mb = float(1000 * 1000) - sys.stderr.write("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) - - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - - # collection - it should be able to collect all - assert man.num_file_handles() - assert man.collect() - assert man.num_file_handles() == 0 - #END for each item - # END for each manager type - os.close(fd) + + def test_cursor(self): + fc = FileCreator(self.k_window_test_size, "cursor_test") + + man = SlidingWindowMapManager() + ci = WindowCursor(man) # invalid cursor + assert not ci.is_valid() + assert not ci.is_associated() + assert ci.size() == 0 # this is cached, so we can query it in invalid state + + cv = man.make_cursor(fc.path) + assert not cv.is_valid() # no region mapped yet + assert cv.is_associated()# but it know where to map it from + assert cv.file_size() == fc.size + assert cv.path() == fc.path + + # copy module + cio = copy(cv) + assert not cio.is_valid() and cio.is_associated() + + # assign method + assert not ci.is_associated() + ci.assign(cv) + assert not ci.is_valid() and ci.is_associated() + + # unuse non-existing region is fine + cv.unuse_region() + cv.unuse_region() + + # destruction is fine (even multiple times) + cv._destroy() + WindowCursor(man)._destroy() + + def test_memory_manager(self): + slide_man = SlidingWindowMapManager() + static_man = StaticWindowMapManager() + + for man in (static_man, slide_man): + assert man.num_file_handles() == 0 + assert man.num_open_files() == 0 + winsize_cmp_val = 0 + if isinstance(man, StaticWindowMapManager): + winsize_cmp_val = -1 + #END handle window size + assert man.window_size() > winsize_cmp_val + assert man.mapped_memory_size() == 0 + assert man.max_mapped_memory_size() > 0 + + # collection doesn't raise in 'any' mode + man._collect_lru_region(0) + # doesn't raise if we are within the limit + man._collect_lru_region(10) + + # doesn't fail if we overallocate + assert man._collect_lru_region(sys.maxint) == 0 + + # use a region, verify most basic functionality + fc = FileCreator(self.k_window_test_size, "manager_test") + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + c = man.make_cursor(item) + assert c.path_or_fd() is item + assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] + + if isinstance(item, int): + self.failUnlessRaises(ValueError, c.path) + else: + self.failUnlessRaises(ValueError, c.fd) + #END handle value error + #END for each input + os.close(fd) + # END for each manager type + + def test_memman_operation(self): + # test more access, force it to actually unmap regions + fc = FileCreator(self.k_window_test_size, "manager_operation_test") + data = open(fc.path, 'rb').read() + fd = os.open(fc.path, os.O_RDONLY) + max_num_handles = 15 + #small_size = + for mtype, args in ( (StaticWindowMapManager, (0, fc.size / 3, max_num_handles)), + (SlidingWindowMapManager, (fc.size / 100, fc.size / 3, max_num_handles)),): + for item in (fc.path, fd): + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) + c = man.make_cursor(item) + + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + # window size is 0 for static managers, hence size will be 0. We take that into consideration + size = man.window_size() / 2 + assert c.use_region(base_offset, size).is_valid() + rr = c.region_ref() + assert rr().client_count() == 2 # the manager and the cursor and us + + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr().size() + + #assert c.size() == size # the cursor may overallocate in its static version + assert c.ofs_begin() == base_offset + assert rr().ofs_begin() == 0 # it was aligned and expanded + if man.window_size(): + assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) + else: + assert rr().size() == fc.size + #END ignore static managers which dont use windows and are aligned to file boundaries + + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + + # obtain second window, which spans the first part of the file - it is a still the same window + nsize = (size or fc.size) - 10 + assert c.use_region(0, nsize).is_valid() + assert c.region_ref()() == rr() + assert man.num_file_handles() == 1 + assert c.size() == nsize + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:nsize] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - (size or c.size()) + overshoot + assert c.use_region(base_offset, size).is_valid() + if man.window_size(): + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager + else: + assert c.size() < fc.size + #END ignore static managers which only have one handle per file + rr = c.region_ref() + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + if man.window_size(): + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + #END ignore this for static managers + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 5000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles + while num_random_accesses: + num_random_accesses -= 1 + base_offset = randint(0, fc.size - 1) + + # precondition + if man.window_size(): + assert max_mapped_memory_size >= mapped_memory_size() + #END statics will overshoot, which is fine + assert max_file_handles >= num_file_handles() + assert c.use_region(base_offset, (size or c.size())).is_valid() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset+csize] + memory_read += csize + + assert includes_ofs(base_offset) + assert includes_ofs(base_offset+csize-1) + assert not includes_ofs(base_offset+csize) + # END while we should do an access + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows + mb = float(1000 * 1000) + sys.stderr.write("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 + #END for each item + # END for each manager type + os.close(fd) diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index a9f4b1c08..4e1a5764b 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -1,83 +1,83 @@ from lib import TestBase class TestTutorial(TestBase): - - def test_example(self): - # Memory Managers - ################## - import smmap - # This instance should be globally available in your application - # It is configured to be well suitable for 32-bit or 64 bit applications. - mman = smmap.SlidingWindowMapManager() - - # the manager provides much useful information about its current state - # like the amount of open file handles or the amount of mapped memory - assert mman.num_file_handles() == 0 - assert mman.mapped_memory_size() == 0 - # and many more ... - - # Cursors - ########## - import smmap.test.lib - fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") - - # obtain a cursor to access some file. - c = mman.make_cursor(fc.path) - - # the cursor is now associated with the file, but not yet usable - assert c.is_associated() - assert not c.is_valid() - - # before you can use the cursor, you have to specify a window you want to - # access. The following just says you want as much data as possible starting - # from offset 0. - # To be sure your region could be mapped, query for validity - assert c.use_region().is_valid() # use_region returns self - - # once a region was mapped, you must query its dimension regularly - # to assure you don't try to access its buffer out of its bounds - assert c.size() - c.buffer()[0] # first byte - c.buffer()[1:10] # first 9 bytes - c.buffer()[c.size()-1] # last byte - - # its recommended not to create big slices when feeding the buffer - # into consumers (e.g. struct or zlib). - # Instead, either give the buffer directly, or use pythons buffer command. - buffer(c.buffer(), 1, 9) # first 9 bytes without copying them - - # you can query absolute offsets, and check whether an offset is included - # in the cursor's data. - assert c.ofs_begin() < c.ofs_end() - assert c.includes_ofs(100) - - # If you are over out of bounds with one of your region requests, the - # cursor will be come invalid. It cannot be used in that state - assert not c.use_region(fc.size, 100).is_valid() - # map as much as possible after skipping the first 100 bytes - assert c.use_region(100).is_valid() - - # You can explicitly free cursor resources by unusing the cursor's region - c.unuse_region() - assert not c.is_valid() - - # Buffers - ######### - # Create a default buffer which can operate on the whole file - buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) - - # you can use it right away - assert buf.cursor().is_valid() - - buf[0] # access the first byte - buf[-1] # access the last ten bytes on the file - buf[-10:]# access the last ten bytes - - # If you want to keep the instance between different accesses, use the - # dedicated methods - buf.end_access() - assert not buf.cursor().is_valid() # you cannot use the buffer anymore - assert buf.begin_access(offset=10) # start using the buffer at an offset - - # it will stop using resources automatically once it goes out of scope - + + def test_example(self): + # Memory Managers + ################## + import smmap + # This instance should be globally available in your application + # It is configured to be well suitable for 32-bit or 64 bit applications. + mman = smmap.SlidingWindowMapManager() + + # the manager provides much useful information about its current state + # like the amount of open file handles or the amount of mapped memory + assert mman.num_file_handles() == 0 + assert mman.mapped_memory_size() == 0 + # and many more ... + + # Cursors + ########## + import smmap.test.lib + fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size()-1] # last byte + + # its recommended not to create big slices when feeding the buffer + # into consumers (e.g. struct or zlib). + # Instead, either give the buffer directly, or use pythons buffer command. + buffer(c.buffer(), 1, 9) # first 9 bytes without copying them + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + # Buffers + ######### + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:]# access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset + + # it will stop using resources automatically once it goes out of scope + diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 096c5f6df..2df0660be 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -6,109 +6,109 @@ import sys class TestMMan(TestBase): - - def test_window(self): - wl = MapWindow(0, 1) # left - wc = MapWindow(1, 1) # center - wc2 = MapWindow(10, 5) # another center - wr = MapWindow(8000, 50) # right - - assert wl.ofs_end() == 1 - assert wc.ofs_end() == 2 - assert wr.ofs_end() == 8050 - - # extension does nothing if already in place - maxsize = 100 - wc.extend_left_to(wl, maxsize) - assert wc.ofs == 1 and wc.size == 1 - wl.extend_right_to(wc, maxsize) - wl.extend_right_to(wc, maxsize) - assert wl.ofs == 0 and wl.size == 1 - - # an actual left extension - pofs_end = wc2.ofs_end() - wc2.extend_left_to(wc, maxsize) - assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() - - - # respects maxsize - wc.extend_right_to(wr, maxsize) - assert wc.ofs == 1 and wc.size == maxsize - wc.extend_right_to(wr, maxsize) - assert wc.ofs == 1 and wc.size == maxsize - - # without maxsize - wc.extend_right_to(wr, sys.maxint) - assert wc.ofs_end() == wr.ofs and wc.ofs == 1 - - # extend left - wr.extend_left_to(wc2, maxsize) - wr.extend_left_to(wc2, maxsize) - assert wr.size == maxsize - - wr.extend_left_to(wc2, sys.maxint) - assert wr.ofs == wc2.ofs_end() - - wc.align() - assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) - - def test_region(self): - fc = FileCreator(self.k_window_test_size, "window_test") - half_size = fc.size / 2 - rofs = align_to_mmap(4200, False) - rfull = MapRegion(fc.path, 0, fc.size) - rhalfofs = MapRegion(fc.path, rofs, fc.size) - rhalfsize = MapRegion(fc.path, 0, half_size) - - # offsets - assert rfull.ofs_begin() == 0 and rfull.size() == fc.size - assert rfull.ofs_end() == fc.size # if this method works, it works always - - assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs - assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size - - assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) - assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) - # with the values we have, this test only works on windows where an alignment - # size of 4096 is assumed. - # We only test on linux as it is inconsitent between the python versions - # as they use different mapping techniques to circumvent the missing offset - # argument of mmap. - if sys.platform != 'win32': - assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) - #END handle platforms - - # auto-refcount - assert rfull.client_count() == 1 - rfull2 = rfull - assert rfull.client_count() == 2 - - # usage - assert rfull.usage_count() == 0 - rfull.increment_usage_count() - assert rfull.usage_count() == 1 - - # window constructor - w = MapWindow.from_region(rfull) - assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() - - def test_region_list(self): - fc = FileCreator(100, "sample_file") - - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - ml = MapRegionList(item) - - assert ml.client_count() == 1 - - assert len(ml) == 0 - assert ml.path_or_fd() == item - assert ml.file_size() == fc.size - #END handle input - os.close(fd) - - def test_util(self): - assert isinstance(is_64_bit(), bool) # just call it - assert align_to_mmap(1, False) == 0 - assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY - + + def test_window(self): + wl = MapWindow(0, 1) # left + wc = MapWindow(1, 1) # center + wc2 = MapWindow(10, 5) # another center + wr = MapWindow(8000, 50) # right + + assert wl.ofs_end() == 1 + assert wc.ofs_end() == 2 + assert wr.ofs_end() == 8050 + + # extension does nothing if already in place + maxsize = 100 + wc.extend_left_to(wl, maxsize) + assert wc.ofs == 1 and wc.size == 1 + wl.extend_right_to(wc, maxsize) + wl.extend_right_to(wc, maxsize) + assert wl.ofs == 0 and wl.size == 1 + + # an actual left extension + pofs_end = wc2.ofs_end() + wc2.extend_left_to(wc, maxsize) + assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() + + + # respects maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + wc.extend_right_to(wr, maxsize) + assert wc.ofs == 1 and wc.size == maxsize + + # without maxsize + wc.extend_right_to(wr, sys.maxint) + assert wc.ofs_end() == wr.ofs and wc.ofs == 1 + + # extend left + wr.extend_left_to(wc2, maxsize) + wr.extend_left_to(wc2, maxsize) + assert wr.size == maxsize + + wr.extend_left_to(wc2, sys.maxint) + assert wr.ofs == wc2.ofs_end() + + wc.align() + assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) + + def test_region(self): + fc = FileCreator(self.k_window_test_size, "window_test") + half_size = fc.size / 2 + rofs = align_to_mmap(4200, False) + rfull = MapRegion(fc.path, 0, fc.size) + rhalfofs = MapRegion(fc.path, rofs, fc.size) + rhalfsize = MapRegion(fc.path, 0, half_size) + + # offsets + assert rfull.ofs_begin() == 0 and rfull.size() == fc.size + assert rfull.ofs_end() == fc.size # if this method works, it works always + + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs + assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size + + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) + # with the values we have, this test only works on windows where an alignment + # size of 4096 is assumed. + # We only test on linux as it is inconsitent between the python versions + # as they use different mapping techniques to circumvent the missing offset + # argument of mmap. + if sys.platform != 'win32': + assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) + #END handle platforms + + # auto-refcount + assert rfull.client_count() == 1 + rfull2 = rfull + assert rfull.client_count() == 2 + + # usage + assert rfull.usage_count() == 0 + rfull.increment_usage_count() + assert rfull.usage_count() == 1 + + # window constructor + w = MapWindow.from_region(rfull) + assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() + + def test_region_list(self): + fc = FileCreator(100, "sample_file") + + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + ml = MapRegionList(item) + + assert ml.client_count() == 1 + + assert len(ml) == 0 + assert ml.path_or_fd() == item + assert ml.file_size() == fc.size + #END handle input + os.close(fd) + + def test_util(self): + assert isinstance(is_64_bit(), bool) # just call it + assert align_to_mmap(1, False) == 0 + assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY + diff --git a/smmap/util.py b/smmap/util.py index b0fd83b3f..c6710b3fe 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -5,36 +5,36 @@ from mmap import mmap, ACCESS_READ try: - from mmap import ALLOCATIONGRANULARITY + from mmap import ALLOCATIONGRANULARITY except ImportError: - # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly - # useful for aligning the offset. The offset argument doesn't exist there though - from mmap import PAGESIZE as ALLOCATIONGRANULARITY + # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly + # useful for aligning the offset. The offset argument doesn't exist there though + from mmap import PAGESIZE as ALLOCATIONGRANULARITY #END handle pythons missing quality assurance from sys import getrefcount -__all__ = [ "align_to_mmap", "is_64_bit", - "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] +__all__ = [ "align_to_mmap", "is_64_bit", + "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities def align_to_mmap(num, round_up): - """ - Align the given integer number to the closest page offset, which usually is 4096 bytes. - - :param round_up: if True, the next higher multiple of page size is used, otherwise - the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) - :return: num rounded to closest page""" - res = (num / ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; - if round_up and (res != num): - res += ALLOCATIONGRANULARITY - #END handle size - return res; - + """ + Align the given integer number to the closest page offset, which usually is 4096 bytes. + + :param round_up: if True, the next higher multiple of page size is used, otherwise + the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) + :return: num rounded to closest page""" + res = (num / ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; + if round_up and (res != num): + res += ALLOCATIONGRANULARITY + #END handle size + return res; + def is_64_bit(): - """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" - return sys.maxint > (1<<32) - 1 + """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" + return sys.maxint > (1<<32) - 1 #}END utilities @@ -42,228 +42,228 @@ def is_64_bit(): #{ Utility Classes class MapWindow(object): - """Utility type which is used to snap windows towards each other, and to adjust their size""" - __slots__ = ( - 'ofs', # offset into the file in bytes - 'size' # size of the window in bytes - ) + """Utility type which is used to snap windows towards each other, and to adjust their size""" + __slots__ = ( + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) - def __init__(self, offset, size): - self.ofs = offset - self.size = size + def __init__(self, offset, size): + self.ofs = offset + self.size = size - def __repr__(self): - return "MapWindow(%i, %i)" % (self.ofs, self.size) + def __repr__(self): + return "MapWindow(%i, %i)" % (self.ofs, self.size) - @classmethod - def from_region(cls, region): - """:return: new window from a region""" - return cls(region._b, region.size()) + @classmethod + def from_region(cls, region): + """:return: new window from a region""" + return cls(region._b, region.size()) - def ofs_end(self): - return self.ofs + self.size + def ofs_end(self): + return self.ofs + self.size - def align(self): - """Assures the previous window area is contained in the new one""" - nofs = align_to_mmap(self.ofs, 0) - self.size += self.ofs - nofs # keep size constant - self.ofs = nofs - self.size = align_to_mmap(self.size, 1) + def align(self): + """Assures the previous window area is contained in the new one""" + nofs = align_to_mmap(self.ofs, 0) + self.size += self.ofs - nofs # keep size constant + self.ofs = nofs + self.size = align_to_mmap(self.size, 1) - def extend_left_to(self, window, max_size): - """Adjust the offset to start where the given window on our left ends if possible, - but don't make yourself larger than max_size. - The resize will assure that the new window still contains the old window area""" - rofs = self.ofs - window.ofs_end() - nsize = rofs + self.size - rofs -= nsize - min(nsize, max_size) - self.ofs = self.ofs - rofs - self.size += rofs + def extend_left_to(self, window, max_size): + """Adjust the offset to start where the given window on our left ends if possible, + but don't make yourself larger than max_size. + The resize will assure that the new window still contains the old window area""" + rofs = self.ofs - window.ofs_end() + nsize = rofs + self.size + rofs -= nsize - min(nsize, max_size) + self.ofs = self.ofs - rofs + self.size += rofs - def extend_right_to(self, window, max_size): - """Adjust the size to make our window end where the right window begins, but don't - get larger than max_size""" - self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) + def extend_right_to(self, window, max_size): + """Adjust the size to make our window end where the right window begins, but don't + get larger than max_size""" + self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) class MapRegion(object): - """Defines a mapped region of memory, aligned to pagesizes - - **Note:** deallocates used region automatically on destruction""" - __slots__ = [ - '_b' , # beginning of mapping - '_mf', # mapped memory chunk (as returned by mmap) - '_uc', # total amount of usages - '_size', # cached size of our memory map - '__weakref__' - ] - _need_compat_layer = sys.version_info[1] < 6 - - if _need_compat_layer: - __slots__.append('_mfb') # mapped memory buffer to provide offset - #END handle additional slot - - #{ Configuration - # Used for testing only. If True, all data will be loaded into memory at once. - # This makes sure no file handles will remain open. - _test_read_into_memory = False - #} END configuration - - - def __init__(self, path_or_fd, ofs, size, flags = 0): - """Initialize a region, allocate the memory map - :param path_or_fd: path to the file to map, or the opened file descriptor - :param ofs: **aligned** offset into the file to be mapped - :param size: if size is larger then the file on disk, the whole file will be - allocated the the size automatically adjusted - :param flags: additional flags to be given when opening the file. - :raise Exception: if no memory can be allocated""" - self._b = ofs - self._size = 0 - self._uc = 0 - - if isinstance(path_or_fd, int): - fd = path_or_fd - else: - fd = os.open(path_or_fd, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) - #END handle fd - - try: - kwargs = dict(access=ACCESS_READ, offset=ofs) - corrected_size = size - sizeofs = ofs - if self._need_compat_layer: - del(kwargs['offset']) - corrected_size += ofs - sizeofs = 0 - # END handle python not supporting offset ! Arg - - # have to correct size, otherwise (instead of the c version) it will - # bark that the size is too large ... many extra file accesses because - # if this ... argh ! - actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) - if self._test_read_into_memory: - self._mf = self._read_into_memory(fd, ofs, actual_size) - else: - self._mf = mmap(fd, actual_size, **kwargs) - #END handle memory mode - - self._size = len(self._mf) - - if self._need_compat_layer: - self._mfb = buffer(self._mf, ofs, self._size) - #END handle buffer wrapping - finally: - if isinstance(path_or_fd, basestring): - os.close(fd) - #END only close it if we opened it - #END close file handle - - def _read_into_memory(self, fd, offset, size): - """:return: string data as read from the given file descriptor, offset and size """ - os.lseek(fd, offset, os.SEEK_SET) - mf = '' - bytes_todo = size - while bytes_todo: - chunk = 1024*1024 - d = os.read(fd, chunk) - bytes_todo -= len(d) - mf += d - #END loop copy items - return mf - - def __repr__(self): - return "MapRegion<%i, %i>" % (self._b, self.size()) - - #{ Interface + """Defines a mapped region of memory, aligned to pagesizes + + **Note:** deallocates used region automatically on destruction""" + __slots__ = [ + '_b' , # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_uc', # total amount of usages + '_size', # cached size of our memory map + '__weakref__' + ] + _need_compat_layer = sys.version_info[1] < 6 + + if _need_compat_layer: + __slots__.append('_mfb') # mapped memory buffer to provide offset + #END handle additional slot + + #{ Configuration + # Used for testing only. If True, all data will be loaded into memory at once. + # This makes sure no file handles will remain open. + _test_read_into_memory = False + #} END configuration + + + def __init__(self, path_or_fd, ofs, size, flags = 0): + """Initialize a region, allocate the memory map + :param path_or_fd: path to the file to map, or the opened file descriptor + :param ofs: **aligned** offset into the file to be mapped + :param size: if size is larger then the file on disk, the whole file will be + allocated the the size automatically adjusted + :param flags: additional flags to be given when opening the file. + :raise Exception: if no memory can be allocated""" + self._b = ofs + self._size = 0 + self._uc = 0 + + if isinstance(path_or_fd, int): + fd = path_or_fd + else: + fd = os.open(path_or_fd, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + #END handle fd + + try: + kwargs = dict(access=ACCESS_READ, offset=ofs) + corrected_size = size + sizeofs = ofs + if self._need_compat_layer: + del(kwargs['offset']) + corrected_size += ofs + sizeofs = 0 + # END handle python not supporting offset ! Arg + + # have to correct size, otherwise (instead of the c version) it will + # bark that the size is too large ... many extra file accesses because + # if this ... argh ! + actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) + if self._test_read_into_memory: + self._mf = self._read_into_memory(fd, ofs, actual_size) + else: + self._mf = mmap(fd, actual_size, **kwargs) + #END handle memory mode + + self._size = len(self._mf) + + if self._need_compat_layer: + self._mfb = buffer(self._mf, ofs, self._size) + #END handle buffer wrapping + finally: + if isinstance(path_or_fd, basestring): + os.close(fd) + #END only close it if we opened it + #END close file handle + + def _read_into_memory(self, fd, offset, size): + """:return: string data as read from the given file descriptor, offset and size """ + os.lseek(fd, offset, os.SEEK_SET) + mf = '' + bytes_todo = size + while bytes_todo: + chunk = 1024*1024 + d = os.read(fd, chunk) + bytes_todo -= len(d) + mf += d + #END loop copy items + return mf + + def __repr__(self): + return "MapRegion<%i, %i>" % (self._b, self.size()) + + #{ Interface - def buffer(self): - """:return: a buffer containing the memory""" - return self._mf - - def map(self): - """:return: a memory map containing the memory""" - return self._mf - - def ofs_begin(self): - """:return: absolute byte offset to the first byte of the mapping""" - return self._b - - def size(self): - """:return: total size of the mapped region in bytes""" - return self._size - - def ofs_end(self): - """:return: Absolute offset to one byte beyond the mapping into the file""" - return self._b + self._size - - def includes_ofs(self, ofs): - """:return: True if the given offset can be read in our mapped region""" - return self._b <= ofs < self._b + self._size - - def client_count(self): - """:return: number of clients currently using this region""" - # -1: self on stack, -1 self in this method, -1 self in getrefcount - return getrefcount(self)-3 - - def usage_count(self): - """:return: amount of usages so far""" - return self._uc - - def increment_usage_count(self): - """Adjust the usage count by the given positive or negative offset""" - self._uc += 1 - - # re-define all methods which need offset adjustments in compatibility mode - if _need_compat_layer: - def size(self): - return self._size - self._b - - def ofs_end(self): - # always the size - we are as large as it gets - return self._size - - def buffer(self): - return self._mfb - - def includes_ofs(self, ofs): - return self._b <= ofs < self._size - #END handle compat layer - - #} END interface - - + def buffer(self): + """:return: a buffer containing the memory""" + return self._mf + + def map(self): + """:return: a memory map containing the memory""" + return self._mf + + def ofs_begin(self): + """:return: absolute byte offset to the first byte of the mapping""" + return self._b + + def size(self): + """:return: total size of the mapped region in bytes""" + return self._size + + def ofs_end(self): + """:return: Absolute offset to one byte beyond the mapping into the file""" + return self._b + self._size + + def includes_ofs(self, ofs): + """:return: True if the given offset can be read in our mapped region""" + return self._b <= ofs < self._b + self._size + + def client_count(self): + """:return: number of clients currently using this region""" + # -1: self on stack, -1 self in this method, -1 self in getrefcount + return getrefcount(self)-3 + + def usage_count(self): + """:return: amount of usages so far""" + return self._uc + + def increment_usage_count(self): + """Adjust the usage count by the given positive or negative offset""" + self._uc += 1 + + # re-define all methods which need offset adjustments in compatibility mode + if _need_compat_layer: + def size(self): + return self._size - self._b + + def ofs_end(self): + # always the size - we are as large as it gets + return self._size + + def buffer(self): + return self._mfb + + def includes_ofs(self, ofs): + return self._b <= ofs < self._size + #END handle compat layer + + #} END interface + + class MapRegionList(list): - """List of MapRegion instances associating a path with a list of regions.""" - __slots__ = ( - '_path_or_fd', # path or file descriptor which is mapped by all our regions - '_file_size' # total size of the file we map - ) - - def __new__(cls, path): - return super(MapRegionList, cls).__new__(cls) - - def __init__(self, path_or_fd): - self._path_or_fd = path_or_fd - self._file_size = None - - def client_count(self): - """:return: amount of clients which hold a reference to this instance""" - return getrefcount(self)-3 - - def path_or_fd(self): - """:return: path or file descriptor we are attached to""" - return self._path_or_fd - - def file_size(self): - """:return: size of file we manager""" - if self._file_size is None: - if isinstance(self._path_or_fd, basestring): - self._file_size = os.stat(self._path_or_fd).st_size - else: - self._file_size = os.fstat(self._path_or_fd).st_size - #END handle path type - #END update file size - return self._file_size - + """List of MapRegion instances associating a path with a list of regions.""" + __slots__ = ( + '_path_or_fd', # path or file descriptor which is mapped by all our regions + '_file_size' # total size of the file we map + ) + + def __new__(cls, path): + return super(MapRegionList, cls).__new__(cls) + + def __init__(self, path_or_fd): + self._path_or_fd = path_or_fd + self._file_size = None + + def client_count(self): + """:return: amount of clients which hold a reference to this instance""" + return getrefcount(self)-3 + + def path_or_fd(self): + """:return: path or file descriptor we are attached to""" + return self._path_or_fd + + def file_size(self): + """:return: size of file we manager""" + if self._file_size is None: + if isinstance(self._path_or_fd, basestring): + self._file_size = os.stat(self._path_or_fd).st_size + else: + self._file_size = os.fstat(self._path_or_fd).st_size + #END handle path type + #END update file size + return self._file_size + #} END utilty classes From 6576d5503a64d124fd7bcf639cc8955918b3ac43 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 9 Feb 2014 20:51:43 +0100 Subject: [PATCH 0204/1392] tabs to spaces --- gitdb/__init__.py | 22 +- gitdb/base.py | 556 ++--- gitdb/db/base.py | 586 ++--- gitdb/db/git.py | 134 +- gitdb/db/loose.py | 470 ++-- gitdb/db/mem.py | 188 +- gitdb/db/pack.py | 374 ++-- gitdb/db/ref.py | 138 +- gitdb/exc.py | 30 +- gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- gitdb/fun.py | 1206 +++++------ gitdb/pack.py | 1920 ++++++++--------- gitdb/stream.py | 1288 +++++------ gitdb/test/__init__.py | 8 +- gitdb/test/db/lib.py | 376 ++-- gitdb/test/db/test_git.py | 74 +- gitdb/test/db/test_loose.py | 50 +- gitdb/test/db/test_mem.py | 46 +- gitdb/test/db/test_pack.py | 118 +- gitdb/test/db/test_ref.py | 98 +- gitdb/test/lib.py | 206 +- gitdb/test/performance/lib.py | 56 +- gitdb/test/performance/test_pack.py | 150 +- gitdb/test/performance/test_pack_streaming.py | 120 +- gitdb/test/performance/test_stream.py | 324 +-- gitdb/test/test_base.py | 168 +- gitdb/test/test_example.py | 102 +- gitdb/test/test_pack.py | 438 ++-- gitdb/test/test_stream.py | 266 +-- gitdb/test/test_util.py | 184 +- gitdb/util.py | 538 ++--- 32 files changed, 5119 insertions(+), 5119 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 800b292da..ff750d14c 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -9,17 +9,17 @@ #{ Initialization def _init_externals(): - """Initialize external projects by putting them into the path""" - for module in ('async', 'smmap'): - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) - - try: - __import__(module) - except ImportError: - raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) - #END verify import - #END handel imports - + """Initialize external projects by putting them into the path""" + for module in ('async', 'smmap'): + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) + + try: + __import__(module) + except ImportError: + raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) + #END verify import + #END handel imports + #} END initialization _init_externals() diff --git a/gitdb/base.py b/gitdb/base.py index ff1062bf6..bad5f7472 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -4,308 +4,308 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with basic data structures - they are designed to be lightweight and fast""" from util import ( - bin_to_hex, - zlib - ) + bin_to_hex, + zlib + ) from fun import ( - type_id_to_type_map, - type_to_type_id_map - ) + type_id_to_type_map, + type_to_type_id_map + ) __all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', - 'OStream', 'OPackStream', 'ODeltaPackStream', - 'IStream', 'InvalidOInfo', 'InvalidOStream' ) + 'OStream', 'OPackStream', 'ODeltaPackStream', + 'IStream', 'InvalidOInfo', 'InvalidOStream' ) #{ ODB Bases class OInfo(tuple): - """Carries information about an object in an ODB, provding information - about the binary sha of the object, the type_string as well as the uncompressed size - in bytes. - - It can be accessed using tuple notation and using attribute access notation:: - - assert dbi[0] == dbi.binsha - assert dbi[1] == dbi.type - assert dbi[2] == dbi.size - - The type is designed to be as lighteight as possible.""" - __slots__ = tuple() - - def __new__(cls, sha, type, size): - return tuple.__new__(cls, (sha, type, size)) - - def __init__(self, *args): - tuple.__init__(self) - - #{ Interface - @property - def binsha(self): - """:return: our sha as binary, 20 bytes""" - return self[0] - - @property - def hexsha(self): - """:return: our sha, hex encoded, 40 bytes""" - return bin_to_hex(self[0]) - - @property - def type(self): - return self[1] - - @property - def type_id(self): - return type_to_type_id_map[self[1]] - - @property - def size(self): - return self[2] - #} END interface - - + """Carries information about an object in an ODB, provding information + about the binary sha of the object, the type_string as well as the uncompressed size + in bytes. + + It can be accessed using tuple notation and using attribute access notation:: + + assert dbi[0] == dbi.binsha + assert dbi[1] == dbi.type + assert dbi[2] == dbi.size + + The type is designed to be as lighteight as possible.""" + __slots__ = tuple() + + def __new__(cls, sha, type, size): + return tuple.__new__(cls, (sha, type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + @property + def binsha(self): + """:return: our sha as binary, 20 bytes""" + return self[0] + + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return bin_to_hex(self[0]) + + @property + def type(self): + return self[1] + + @property + def type_id(self): + return type_to_type_id_map[self[1]] + + @property + def size(self): + return self[2] + #} END interface + + class OPackInfo(tuple): - """As OInfo, but provides a type_id property to retrieve the numerical type id, and - does not include a sha. - - Additionally, the pack_offset is the absolute offset into the packfile at which - all object information is located. The data_offset property points to the abosolute - location in the pack at which that actual data stream can be found.""" - __slots__ = tuple() - - def __new__(cls, packoffset, type, size): - return tuple.__new__(cls, (packoffset,type, size)) - - def __init__(self, *args): - tuple.__init__(self) - - #{ Interface - - @property - def pack_offset(self): - return self[0] - - @property - def type(self): - return type_id_to_type_map[self[1]] - - @property - def type_id(self): - return self[1] - - @property - def size(self): - return self[2] - - #} END interface - - + """As OInfo, but provides a type_id property to retrieve the numerical type id, and + does not include a sha. + + Additionally, the pack_offset is the absolute offset into the packfile at which + all object information is located. The data_offset property points to the abosolute + location in the pack at which that actual data stream can be found.""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size): + return tuple.__new__(cls, (packoffset,type, size)) + + def __init__(self, *args): + tuple.__init__(self) + + #{ Interface + + @property + def pack_offset(self): + return self[0] + + @property + def type(self): + return type_id_to_type_map[self[1]] + + @property + def type_id(self): + return self[1] + + @property + def size(self): + return self[2] + + #} END interface + + class ODeltaPackInfo(OPackInfo): - """Adds delta specific information, - Either the 20 byte sha which points to some object in the database, - or the negative offset from the pack_offset, so that pack_offset - delta_info yields - the pack offset of the base object""" - __slots__ = tuple() - - def __new__(cls, packoffset, type, size, delta_info): - return tuple.__new__(cls, (packoffset, type, size, delta_info)) - - #{ Interface - @property - def delta_info(self): - return self[3] - #} END interface - - + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, + or the negative offset from the pack_offset, so that pack_offset - delta_info yields + the pack offset of the base object""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, delta_info): + return tuple.__new__(cls, (packoffset, type, size, delta_info)) + + #{ Interface + @property + def delta_info(self): + return self[3] + #} END interface + + class OStream(OInfo): - """Base for object streams retrieved from the database, providing additional - information about the stream. - Generally, ODB streams are read-only as objects are immutable""" - __slots__ = tuple() - - def __new__(cls, sha, type, size, stream, *args, **kwargs): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (sha, type, size, stream)) - - - def __init__(self, *args, **kwargs): - tuple.__init__(self) - - #{ Stream Reader Interface - - def read(self, size=-1): - return self[3].read(size) - - @property - def stream(self): - return self[3] - - #} END stream reader interface - - + """Base for object streams retrieved from the database, providing additional + information about the stream. + Generally, ODB streams are read-only as objects are immutable""" + __slots__ = tuple() + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + + def __init__(self, *args, **kwargs): + tuple.__init__(self) + + #{ Stream Reader Interface + + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + + #} END stream reader interface + + class ODeltaStream(OStream): - """Uses size info of its stream, delaying reads""" - - def __new__(cls, sha, type, size, stream, *args, **kwargs): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (sha, type, size, stream)) - - #{ Stream Reader Interface - - @property - def size(self): - return self[3].size - - #} END stream reader interface - - + """Uses size info of its stream, delaying reads""" + + def __new__(cls, sha, type, size, stream, *args, **kwargs): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (sha, type, size, stream)) + + #{ Stream Reader Interface + + @property + def size(self): + return self[3].size + + #} END stream reader interface + + class OPackStream(OPackInfo): - """Next to pack object information, a stream outputting an undeltified base object - is provided""" - __slots__ = tuple() - - def __new__(cls, packoffset, type, size, stream, *args): - """Helps with the initialization of subclasses""" - return tuple.__new__(cls, (packoffset, type, size, stream)) - - #{ Stream Reader Interface - def read(self, size=-1): - return self[3].read(size) - - @property - def stream(self): - return self[3] - #} END stream reader interface + """Next to pack object information, a stream outputting an undeltified base object + is provided""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, stream, *args): + """Helps with the initialization of subclasses""" + return tuple.__new__(cls, (packoffset, type, size, stream)) + + #{ Stream Reader Interface + def read(self, size=-1): + return self[3].read(size) + + @property + def stream(self): + return self[3] + #} END stream reader interface - + class ODeltaPackStream(ODeltaPackInfo): - """Provides a stream outputting the uncompressed offset delta information""" - __slots__ = tuple() - - def __new__(cls, packoffset, type, size, delta_info, stream): - return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) + """Provides a stream outputting the uncompressed offset delta information""" + __slots__ = tuple() + + def __new__(cls, packoffset, type, size, delta_info, stream): + return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) - #{ Stream Reader Interface - def read(self, size=-1): - return self[4].read(size) - - @property - def stream(self): - return self[4] - #} END stream reader interface + #{ Stream Reader Interface + def read(self, size=-1): + return self[4].read(size) + + @property + def stream(self): + return self[4] + #} END stream reader interface class IStream(list): - """Represents an input content stream to be fed into the ODB. It is mutable to allow - the ODB to record information about the operations outcome right in this instance. - - It provides interfaces for the OStream and a StreamReader to allow the instance - to blend in without prior conversion. - - The only method your content stream must support is 'read'""" - __slots__ = tuple() - - def __new__(cls, type, size, stream, sha=None): - return list.__new__(cls, (sha, type, size, stream, None)) - - def __init__(self, type, size, stream, sha=None): - list.__init__(self, (sha, type, size, stream, None)) - - #{ Interface - @property - def hexsha(self): - """:return: our sha, hex encoded, 40 bytes""" - return bin_to_hex(self[0]) - - def _error(self): - """:return: the error that occurred when processing the stream, or None""" - return self[4] - - def _set_error(self, exc): - """Set this input stream to the given exc, may be None to reset the error""" - self[4] = exc - - error = property(_error, _set_error) - - #} END interface - - #{ Stream Reader Interface - - def read(self, size=-1): - """Implements a simple stream reader interface, passing the read call on - to our internal stream""" - return self[3].read(size) - - #} END stream reader interface - - #{ interface - - def _set_binsha(self, binsha): - self[0] = binsha - - def _binsha(self): - return self[0] - - binsha = property(_binsha, _set_binsha) - - - def _type(self): - return self[1] - - def _set_type(self, type): - self[1] = type - - type = property(_type, _set_type) - - def _size(self): - return self[2] - - def _set_size(self, size): - self[2] = size - - size = property(_size, _set_size) - - def _stream(self): - return self[3] - - def _set_stream(self, stream): - self[3] = stream - - stream = property(_stream, _set_stream) - - #} END odb info interface - + """Represents an input content stream to be fed into the ODB. It is mutable to allow + the ODB to record information about the operations outcome right in this instance. + + It provides interfaces for the OStream and a StreamReader to allow the instance + to blend in without prior conversion. + + The only method your content stream must support is 'read'""" + __slots__ = tuple() + + def __new__(cls, type, size, stream, sha=None): + return list.__new__(cls, (sha, type, size, stream, None)) + + def __init__(self, type, size, stream, sha=None): + list.__init__(self, (sha, type, size, stream, None)) + + #{ Interface + @property + def hexsha(self): + """:return: our sha, hex encoded, 40 bytes""" + return bin_to_hex(self[0]) + + def _error(self): + """:return: the error that occurred when processing the stream, or None""" + return self[4] + + def _set_error(self, exc): + """Set this input stream to the given exc, may be None to reset the error""" + self[4] = exc + + error = property(_error, _set_error) + + #} END interface + + #{ Stream Reader Interface + + def read(self, size=-1): + """Implements a simple stream reader interface, passing the read call on + to our internal stream""" + return self[3].read(size) + + #} END stream reader interface + + #{ interface + + def _set_binsha(self, binsha): + self[0] = binsha + + def _binsha(self): + return self[0] + + binsha = property(_binsha, _set_binsha) + + + def _type(self): + return self[1] + + def _set_type(self, type): + self[1] = type + + type = property(_type, _set_type) + + def _size(self): + return self[2] + + def _set_size(self, size): + self[2] = size + + size = property(_size, _set_size) + + def _stream(self): + return self[3] + + def _set_stream(self, stream): + self[3] = stream + + stream = property(_stream, _set_stream) + + #} END odb info interface + class InvalidOInfo(tuple): - """Carries information about a sha identifying an object which is invalid in - the queried database. The exception attribute provides more information about - the cause of the issue""" - __slots__ = tuple() - - def __new__(cls, sha, exc): - return tuple.__new__(cls, (sha, exc)) - - def __init__(self, sha, exc): - tuple.__init__(self, (sha, exc)) - - @property - def binsha(self): - return self[0] - - @property - def hexsha(self): - return bin_to_hex(self[0]) - - @property - def error(self): - """:return: exception instance explaining the failure""" - return self[1] + """Carries information about a sha identifying an object which is invalid in + the queried database. The exception attribute provides more information about + the cause of the issue""" + __slots__ = tuple() + + def __new__(cls, sha, exc): + return tuple.__new__(cls, (sha, exc)) + + def __init__(self, sha, exc): + tuple.__init__(self, (sha, exc)) + + @property + def binsha(self): + return self[0] + + @property + def hexsha(self): + return bin_to_hex(self[0]) + + @property + def error(self): + """:return: exception instance explaining the failure""" + return self[1] class InvalidOStream(InvalidOInfo): - """Carries information about an invalid ODB stream""" - __slots__ = tuple() - + """Carries information about an invalid ODB stream""" + __slots__ = tuple() + #} END ODB Bases diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 984acafbf..867e93a81 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -4,20 +4,20 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains implementations of database retrieveing objects""" from gitdb.util import ( - pool, - join, - LazyMixin, - hex_to_bin - ) + pool, + join, + LazyMixin, + hex_to_bin + ) from gitdb.exc import ( - BadObject, - AmbiguousObjectName - ) + BadObject, + AmbiguousObjectName + ) from async import ( - ChannelThreadTask - ) + ChannelThreadTask + ) from itertools import chain @@ -26,301 +26,301 @@ class ObjectDBR(object): - """Defines an interface for object database lookup. - Objects are identified either by their 20 byte bin sha""" - - def __contains__(self, sha): - return self.has_obj - - #{ Query Interface - def has_object(self, sha): - """ - :return: True if the object identified by the given 20 bytes - binary sha is contained in the database""" - raise NotImplementedError("To be implemented in subclass") - - def has_object_async(self, reader): - """Return a reader yielding information about the membership of objects - as identified by shas - :param reader: Reader yielding 20 byte shas. - :return: async.Reader yielding tuples of (sha, bool) pairs which indicate - whether the given sha exists in the database or not""" - task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) - return pool.add_task(task) - - def info(self, sha): - """ :return: OInfo instance - :param sha: bytes binary sha - :raise BadObject:""" - raise NotImplementedError("To be implemented in subclass") - - def info_async(self, reader): - """Retrieve information of a multitude of objects asynchronously - :param reader: Channel yielding the sha's of the objects of interest - :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" - task = ChannelThreadTask(reader, str(self.info_async), self.info) - return pool.add_task(task) - - def stream(self, sha): - """:return: OStream instance - :param sha: 20 bytes binary sha - :raise BadObject:""" - raise NotImplementedError("To be implemented in subclass") - - def stream_async(self, reader): - """Retrieve the OStream of multiple objects - :param reader: see ``info`` - :param max_threads: see ``ObjectDBW.store`` - :return: async.Reader yielding OStream|InvalidOStream instances in any order - - **Note:** depending on the system configuration, it might not be possible to - read all OStreams at once. Instead, read them individually using reader.read(x) - where x is small enough.""" - # base implementation just uses the stream method repeatedly - task = ChannelThreadTask(reader, str(self.stream_async), self.stream) - return pool.add_task(task) - - def size(self): - """:return: amount of objects in this database""" - raise NotImplementedError() - - def sha_iter(self): - """Return iterator yielding 20 byte shas for all objects in this data base""" - raise NotImplementedError() - - #} END query interface - - + """Defines an interface for object database lookup. + Objects are identified either by their 20 byte bin sha""" + + def __contains__(self, sha): + return self.has_obj + + #{ Query Interface + def has_object(self, sha): + """ + :return: True if the object identified by the given 20 bytes + binary sha is contained in the database""" + raise NotImplementedError("To be implemented in subclass") + + def has_object_async(self, reader): + """Return a reader yielding information about the membership of objects + as identified by shas + :param reader: Reader yielding 20 byte shas. + :return: async.Reader yielding tuples of (sha, bool) pairs which indicate + whether the given sha exists in the database or not""" + task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) + return pool.add_task(task) + + def info(self, sha): + """ :return: OInfo instance + :param sha: bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def info_async(self, reader): + """Retrieve information of a multitude of objects asynchronously + :param reader: Channel yielding the sha's of the objects of interest + :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" + task = ChannelThreadTask(reader, str(self.info_async), self.info) + return pool.add_task(task) + + def stream(self, sha): + """:return: OStream instance + :param sha: 20 bytes binary sha + :raise BadObject:""" + raise NotImplementedError("To be implemented in subclass") + + def stream_async(self, reader): + """Retrieve the OStream of multiple objects + :param reader: see ``info`` + :param max_threads: see ``ObjectDBW.store`` + :return: async.Reader yielding OStream|InvalidOStream instances in any order + + **Note:** depending on the system configuration, it might not be possible to + read all OStreams at once. Instead, read them individually using reader.read(x) + where x is small enough.""" + # base implementation just uses the stream method repeatedly + task = ChannelThreadTask(reader, str(self.stream_async), self.stream) + return pool.add_task(task) + + def size(self): + """:return: amount of objects in this database""" + raise NotImplementedError() + + def sha_iter(self): + """Return iterator yielding 20 byte shas for all objects in this data base""" + raise NotImplementedError() + + #} END query interface + + class ObjectDBW(object): - """Defines an interface to create objects in the database""" - - def __init__(self, *args, **kwargs): - self._ostream = None - - #{ Edit Interface - def set_ostream(self, stream): - """ - Adjusts the stream to which all data should be sent when storing new objects - - :param stream: if not None, the stream to use, if None the default stream - will be used. - :return: previously installed stream, or None if there was no override - :raise TypeError: if the stream doesn't have the supported functionality""" - cstream = self._ostream - self._ostream = stream - return cstream - - def ostream(self): - """ - :return: overridden output stream this instance will write to, or None - if it will write to the default stream""" - return self._ostream - - def store(self, istream): - """ - Create a new object in the database - :return: the input istream object with its sha set to its corresponding value - - :param istream: IStream compatible instance. If its sha is already set - to a value, the object will just be stored in the our database format, - in which case the input stream is expected to be in object format ( header + contents ). - :raise IOError: if data could not be written""" - raise NotImplementedError("To be implemented in subclass") - - def store_async(self, reader): - """ - Create multiple new objects in the database asynchronously. The method will - return right away, returning an output channel which receives the results as - they are computed. - - :return: Channel yielding your IStream which served as input, in any order. - The IStreams sha will be set to the sha it received during the process, - or its error attribute will be set to the exception informing about the error. - - :param reader: async.Reader yielding IStream instances. - The same instances will be used in the output channel as were received - in by the Reader. - - **Note:** As some ODB implementations implement this operation atomic, they might - abort the whole operation if one item could not be processed. Hence check how - many items have actually been produced.""" - # base implementation uses store to perform the work - task = ChannelThreadTask(reader, str(self.store_async), self.store) - return pool.add_task(task) - - #} END edit interface - + """Defines an interface to create objects in the database""" + + def __init__(self, *args, **kwargs): + self._ostream = None + + #{ Edit Interface + def set_ostream(self, stream): + """ + Adjusts the stream to which all data should be sent when storing new objects + + :param stream: if not None, the stream to use, if None the default stream + will be used. + :return: previously installed stream, or None if there was no override + :raise TypeError: if the stream doesn't have the supported functionality""" + cstream = self._ostream + self._ostream = stream + return cstream + + def ostream(self): + """ + :return: overridden output stream this instance will write to, or None + if it will write to the default stream""" + return self._ostream + + def store(self, istream): + """ + Create a new object in the database + :return: the input istream object with its sha set to its corresponding value + + :param istream: IStream compatible instance. If its sha is already set + to a value, the object will just be stored in the our database format, + in which case the input stream is expected to be in object format ( header + contents ). + :raise IOError: if data could not be written""" + raise NotImplementedError("To be implemented in subclass") + + def store_async(self, reader): + """ + Create multiple new objects in the database asynchronously. The method will + return right away, returning an output channel which receives the results as + they are computed. + + :return: Channel yielding your IStream which served as input, in any order. + The IStreams sha will be set to the sha it received during the process, + or its error attribute will be set to the exception informing about the error. + + :param reader: async.Reader yielding IStream instances. + The same instances will be used in the output channel as were received + in by the Reader. + + **Note:** As some ODB implementations implement this operation atomic, they might + abort the whole operation if one item could not be processed. Hence check how + many items have actually been produced.""" + # base implementation uses store to perform the work + task = ChannelThreadTask(reader, str(self.store_async), self.store) + return pool.add_task(task) + + #} END edit interface + class FileDBBase(object): - """Provides basic facilities to retrieve files of interest, including - caching facilities to help mapping hexsha's to objects""" - - def __init__(self, root_path): - """Initialize this instance to look for its files at the given root path - All subsequent operations will be relative to this path - :raise InvalidDBRoot: - **Note:** The base will not perform any accessablity checking as the base - might not yet be accessible, but become accessible before the first - access.""" - super(FileDBBase, self).__init__() - self._root_path = root_path - - - #{ Interface - def root_path(self): - """:return: path at which this db operates""" - return self._root_path - - def db_path(self, rela_path): - """ - :return: the given relative path relative to our database root, allowing - to pontentially access datafiles""" - return join(self._root_path, rela_path) - #} END interface - + """Provides basic facilities to retrieve files of interest, including + caching facilities to help mapping hexsha's to objects""" + + def __init__(self, root_path): + """Initialize this instance to look for its files at the given root path + All subsequent operations will be relative to this path + :raise InvalidDBRoot: + **Note:** The base will not perform any accessablity checking as the base + might not yet be accessible, but become accessible before the first + access.""" + super(FileDBBase, self).__init__() + self._root_path = root_path + + + #{ Interface + def root_path(self): + """:return: path at which this db operates""" + return self._root_path + + def db_path(self, rela_path): + """ + :return: the given relative path relative to our database root, allowing + to pontentially access datafiles""" + return join(self._root_path, rela_path) + #} END interface + class CachingDB(object): - """A database which uses caches to speed-up access""" - - #{ Interface - def update_cache(self, force=False): - """ - Call this method if the underlying data changed to trigger an update - of the internal caching structures. - - :param force: if True, the update must be performed. Otherwise the implementation - may decide not to perform an update if it thinks nothing has changed. - :return: True if an update was performed as something change indeed""" - - # END interface + """A database which uses caches to speed-up access""" + + #{ Interface + def update_cache(self, force=False): + """ + Call this method if the underlying data changed to trigger an update + of the internal caching structures. + + :param force: if True, the update must be performed. Otherwise the implementation + may decide not to perform an update if it thinks nothing has changed. + :return: True if an update was performed as something change indeed""" + + # END interface def _databases_recursive(database, output): - """Fill output list with database from db, in order. Deals with Loose, Packed - and compound databases.""" - if isinstance(database, CompoundDB): - compounds = list() - dbs = database.databases() - output.extend(db for db in dbs if not isinstance(db, CompoundDB)) - for cdb in (db for db in dbs if isinstance(db, CompoundDB)): - _databases_recursive(cdb, output) - else: - output.append(database) - # END handle database type - + """Fill output list with database from db, in order. Deals with Loose, Packed + and compound databases.""" + if isinstance(database, CompoundDB): + compounds = list() + dbs = database.databases() + output.extend(db for db in dbs if not isinstance(db, CompoundDB)) + for cdb in (db for db in dbs if isinstance(db, CompoundDB)): + _databases_recursive(cdb, output) + else: + output.append(database) + # END handle database type + class CompoundDB(ObjectDBR, LazyMixin, CachingDB): - """A database which delegates calls to sub-databases. - - Databases are stored in the lazy-loaded _dbs attribute. - Define _set_cache_ to update it with your databases""" - def _set_cache_(self, attr): - if attr == '_dbs': - self._dbs = list() - elif attr == '_db_cache': - self._db_cache = dict() - else: - super(CompoundDB, self)._set_cache_(attr) - - def _db_query(self, sha): - """:return: database containing the given 20 byte sha - :raise BadObject:""" - # most databases use binary representations, prevent converting - # it everytime a database is being queried - try: - return self._db_cache[sha] - except KeyError: - pass - # END first level cache - - for db in self._dbs: - if db.has_object(sha): - self._db_cache[sha] = db - return db - # END for each database - raise BadObject(sha) - - #{ ObjectDBR interface - - def has_object(self, sha): - try: - self._db_query(sha) - return True - except BadObject: - return False - # END handle exceptions - - def info(self, sha): - return self._db_query(sha).info(sha) - - def stream(self, sha): - return self._db_query(sha).stream(sha) + """A database which delegates calls to sub-databases. + + Databases are stored in the lazy-loaded _dbs attribute. + Define _set_cache_ to update it with your databases""" + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + elif attr == '_db_cache': + self._db_cache = dict() + else: + super(CompoundDB, self)._set_cache_(attr) + + def _db_query(self, sha): + """:return: database containing the given 20 byte sha + :raise BadObject:""" + # most databases use binary representations, prevent converting + # it everytime a database is being queried + try: + return self._db_cache[sha] + except KeyError: + pass + # END first level cache + + for db in self._dbs: + if db.has_object(sha): + self._db_cache[sha] = db + return db + # END for each database + raise BadObject(sha) + + #{ ObjectDBR interface + + def has_object(self, sha): + try: + self._db_query(sha) + return True + except BadObject: + return False + # END handle exceptions + + def info(self, sha): + return self._db_query(sha).info(sha) + + def stream(self, sha): + return self._db_query(sha).stream(sha) - def size(self): - """:return: total size of all contained databases""" - return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) - - def sha_iter(self): - return chain(*(db.sha_iter() for db in self._dbs)) - - #} END object DBR Interface - - #{ Interface - - def databases(self): - """:return: tuple of database instances we use for lookups""" - return tuple(self._dbs) + def size(self): + """:return: total size of all contained databases""" + return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) + + def sha_iter(self): + return chain(*(db.sha_iter() for db in self._dbs)) + + #} END object DBR Interface + + #{ Interface + + def databases(self): + """:return: tuple of database instances we use for lookups""" + return tuple(self._dbs) - def update_cache(self, force=False): - # something might have changed, clear everything - self._db_cache.clear() - stat = False - for db in self._dbs: - if isinstance(db, CachingDB): - stat |= db.update_cache(force) - # END if is caching db - # END for each database to update - return stat - - def partial_to_complete_sha_hex(self, partial_hexsha): - """ - :return: 20 byte binary sha1 from the given less-than-40 byte hexsha - :param partial_hexsha: hexsha with less than 40 byte - :raise AmbiguousObjectName: """ - databases = list() - _databases_recursive(self, databases) - - len_partial_hexsha = len(partial_hexsha) - if len_partial_hexsha % 2 != 0: - partial_binsha = hex_to_bin(partial_hexsha + "0") - else: - partial_binsha = hex_to_bin(partial_hexsha) - # END assure successful binary conversion - - candidate = None - for db in databases: - full_bin_sha = None - try: - if hasattr(db, 'partial_to_complete_sha_hex'): - full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) - else: - full_bin_sha = db.partial_to_complete_sha(partial_binsha, len_partial_hexsha) - # END handle database type - except BadObject: - continue - # END ignore bad objects - if full_bin_sha: - if candidate and candidate != full_bin_sha: - raise AmbiguousObjectName(partial_hexsha) - candidate = full_bin_sha - # END handle candidate - # END for each db - if not candidate: - raise BadObject(partial_binsha) - return candidate - - #} END interface - + def update_cache(self, force=False): + # something might have changed, clear everything + self._db_cache.clear() + stat = False + for db in self._dbs: + if isinstance(db, CachingDB): + stat |= db.update_cache(force) + # END if is caching db + # END for each database to update + return stat + + def partial_to_complete_sha_hex(self, partial_hexsha): + """ + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha + :param partial_hexsha: hexsha with less than 40 byte + :raise AmbiguousObjectName: """ + databases = list() + _databases_recursive(self, databases) + + len_partial_hexsha = len(partial_hexsha) + if len_partial_hexsha % 2 != 0: + partial_binsha = hex_to_bin(partial_hexsha + "0") + else: + partial_binsha = hex_to_bin(partial_hexsha) + # END assure successful binary conversion + + candidate = None + for db in databases: + full_bin_sha = None + try: + if hasattr(db, 'partial_to_complete_sha_hex'): + full_bin_sha = db.partial_to_complete_sha_hex(partial_hexsha) + else: + full_bin_sha = db.partial_to_complete_sha(partial_binsha, len_partial_hexsha) + # END handle database type + except BadObject: + continue + # END ignore bad objects + if full_bin_sha: + if candidate and candidate != full_bin_sha: + raise AmbiguousObjectName(partial_hexsha) + candidate = full_bin_sha + # END handle candidate + # END for each db + if not candidate: + raise BadObject(partial_binsha) + return candidate + + #} END interface + diff --git a/gitdb/db/git.py b/gitdb/db/git.py index b8fc46aa0..1d6ad0f26 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -3,10 +3,10 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( - CompoundDB, - ObjectDBW, - FileDBBase - ) + CompoundDB, + ObjectDBW, + FileDBBase + ) from loose import LooseObjectDB from pack import PackedDB @@ -14,72 +14,72 @@ from gitdb.util import LazyMixin from gitdb.exc import ( - InvalidDBRoot, - BadObject, - AmbiguousObjectName - ) + InvalidDBRoot, + BadObject, + AmbiguousObjectName + ) import os __all__ = ('GitDB', ) class GitDB(FileDBBase, ObjectDBW, CompoundDB): - """A git-style object database, which contains all objects in the 'objects' - subdirectory""" - # Configuration - PackDBCls = PackedDB - LooseDBCls = LooseObjectDB - ReferenceDBCls = ReferenceDB - - # Directories - packs_dir = 'pack' - loose_dir = '' - alternates_dir = os.path.join('info', 'alternates') - - def __init__(self, root_path): - """Initialize ourselves on a git objects directory""" - super(GitDB, self).__init__(root_path) - - def _set_cache_(self, attr): - if attr == '_dbs' or attr == '_loose_db': - self._dbs = list() - loose_db = None - for subpath, dbcls in ((self.packs_dir, self.PackDBCls), - (self.loose_dir, self.LooseDBCls), - (self.alternates_dir, self.ReferenceDBCls)): - path = self.db_path(subpath) - if os.path.exists(path): - self._dbs.append(dbcls(path)) - if dbcls is self.LooseDBCls: - loose_db = self._dbs[-1] - # END remember loose db - # END check path exists - # END for each db type - - # should have at least one subdb - if not self._dbs: - raise InvalidDBRoot(self.root_path()) - # END handle error - - # we the first one should have the store method - assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" - - # finally set the value - self._loose_db = loose_db - else: - super(GitDB, self)._set_cache_(attr) - # END handle attrs - - #{ ObjectDBW interface - - def store(self, istream): - return self._loose_db.store(istream) - - def ostream(self): - return self._loose_db.ostream() - - def set_ostream(self, ostream): - return self._loose_db.set_ostream(ostream) - - #} END objectdbw interface - + """A git-style object database, which contains all objects in the 'objects' + subdirectory""" + # Configuration + PackDBCls = PackedDB + LooseDBCls = LooseObjectDB + ReferenceDBCls = ReferenceDB + + # Directories + packs_dir = 'pack' + loose_dir = '' + alternates_dir = os.path.join('info', 'alternates') + + def __init__(self, root_path): + """Initialize ourselves on a git objects directory""" + super(GitDB, self).__init__(root_path) + + def _set_cache_(self, attr): + if attr == '_dbs' or attr == '_loose_db': + self._dbs = list() + loose_db = None + for subpath, dbcls in ((self.packs_dir, self.PackDBCls), + (self.loose_dir, self.LooseDBCls), + (self.alternates_dir, self.ReferenceDBCls)): + path = self.db_path(subpath) + if os.path.exists(path): + self._dbs.append(dbcls(path)) + if dbcls is self.LooseDBCls: + loose_db = self._dbs[-1] + # END remember loose db + # END check path exists + # END for each db type + + # should have at least one subdb + if not self._dbs: + raise InvalidDBRoot(self.root_path()) + # END handle error + + # we the first one should have the store method + assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" + + # finally set the value + self._loose_db = loose_db + else: + super(GitDB, self)._set_cache_(attr) + # END handle attrs + + #{ ObjectDBW interface + + def store(self, istream): + return self._loose_db.store(istream) + + def ostream(self): + return self._loose_db.ostream() + + def set_ostream(self, ostream): + return self._loose_db.set_ostream(ostream) + + #} END objectdbw interface + diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 6cd1cefd5..dc0ea0e3b 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -3,53 +3,53 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( - FileDBBase, - ObjectDBR, - ObjectDBW - ) + FileDBBase, + ObjectDBR, + ObjectDBW + ) from gitdb.exc import ( - InvalidDBRoot, - BadObject, - AmbiguousObjectName - ) + InvalidDBRoot, + BadObject, + AmbiguousObjectName + ) from gitdb.stream import ( - DecompressMemMapReader, - FDCompressedSha1Writer, - FDStream, - Sha1Writer - ) + DecompressMemMapReader, + FDCompressedSha1Writer, + FDStream, + Sha1Writer + ) from gitdb.base import ( - OStream, - OInfo - ) + OStream, + OInfo + ) from gitdb.util import ( - file_contents_ro_filepath, - ENOENT, - hex_to_bin, - bin_to_hex, - exists, - chmod, - isdir, - isfile, - remove, - mkdir, - rename, - dirname, - basename, - join - ) + file_contents_ro_filepath, + ENOENT, + hex_to_bin, + bin_to_hex, + exists, + chmod, + isdir, + isfile, + remove, + mkdir, + rename, + dirname, + basename, + join + ) from gitdb.fun import ( - chunk_size, - loose_object_header_info, - write_object, - stream_copy - ) + chunk_size, + loose_object_header_info, + write_object, + stream_copy + ) import tempfile import mmap @@ -61,202 +61,202 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): - """A database which operates on loose object files""" - - # CONFIGURATION - # chunks in which data will be copied between streams - stream_chunk_size = chunk_size - - # On windows we need to keep it writable, otherwise it cannot be removed - # either - new_objects_mode = 0444 - if os.name == 'nt': - new_objects_mode = 0644 - - - def __init__(self, root_path): - super(LooseObjectDB, self).__init__(root_path) - self._hexsha_to_file = dict() - # Additional Flags - might be set to 0 after the first failure - # Depending on the root, this might work for some mounts, for others not, which - # is why it is per instance - self._fd_open_flags = getattr(os, 'O_NOATIME', 0) - - #{ Interface - def object_path(self, hexsha): - """ - :return: path at which the object with the given hexsha would be stored, - relative to the database root""" - return join(hexsha[:2], hexsha[2:]) - - def readable_db_object_path(self, hexsha): - """ - :return: readable object path to the object identified by hexsha - :raise BadObject: If the object file does not exist""" - try: - return self._hexsha_to_file[hexsha] - except KeyError: - pass - # END ignore cache misses - - # try filesystem - path = self.db_path(self.object_path(hexsha)) - if exists(path): - self._hexsha_to_file[hexsha] = path - return path - # END handle cache - raise BadObject(hexsha) - - def partial_to_complete_sha_hex(self, partial_hexsha): - """:return: 20 byte binary sha1 string which matches the given name uniquely - :param name: hexadecimal partial name - :raise AmbiguousObjectName: - :raise BadObject: """ - candidate = None - for binsha in self.sha_iter(): - if bin_to_hex(binsha).startswith(partial_hexsha): - # it can't ever find the same object twice - if candidate is not None: - raise AmbiguousObjectName(partial_hexsha) - candidate = binsha - # END for each object - if candidate is None: - raise BadObject(partial_hexsha) - return candidate - - #} END interface - - def _map_loose_object(self, sha): - """ - :return: memory map of that file to allow random read access - :raise BadObject: if object could not be located""" - db_path = self.db_path(self.object_path(bin_to_hex(sha))) - try: - return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) - except OSError,e: - if e.errno != ENOENT: - # try again without noatime - try: - return file_contents_ro_filepath(db_path) - except OSError: - raise BadObject(sha) - # didn't work because of our flag, don't try it again - self._fd_open_flags = 0 - else: - raise BadObject(sha) - # END handle error - # END exception handling - try: - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - finally: - os.close(fd) - # END assure file is closed - - def set_ostream(self, stream): - """:raise TypeError: if the stream does not support the Sha1Writer interface""" - if stream is not None and not isinstance(stream, Sha1Writer): - raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) - return super(LooseObjectDB, self).set_ostream(stream) - - def info(self, sha): - m = self._map_loose_object(sha) - try: - type, size = loose_object_header_info(m) - return OInfo(sha, type, size) - finally: - m.close() - # END assure release of system resources - - def stream(self, sha): - m = self._map_loose_object(sha) - type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) - return OStream(sha, type, size, stream) - - def has_object(self, sha): - try: - self.readable_db_object_path(bin_to_hex(sha)) - return True - except BadObject: - return False - # END check existance - - def store(self, istream): - """note: The sha we produce will be hex by nature""" - tmp_path = None - writer = self.ostream() - if writer is None: - # open a tmp file to write the data to - fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - - if istream.binsha is None: - writer = FDCompressedSha1Writer(fd) - else: - writer = FDStream(fd) - # END handle direct stream copies - # END handle custom writer - - try: - try: - if istream.binsha is not None: - # copy as much as possible, the actual uncompressed item size might - # be smaller than the compressed version - stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) - else: - # write object with header, we have to make a new one - write_object(istream.type, istream.size, istream.read, writer.write, - chunk_size=self.stream_chunk_size) - # END handle direct stream copies - finally: - if tmp_path: - writer.close() - # END assure target stream is closed - except: - if tmp_path: - os.remove(tmp_path) - raise - # END assure tmpfile removal on error - - hexsha = None - if istream.binsha: - hexsha = istream.hexsha - else: - hexsha = writer.sha(as_hex=True) - # END handle sha - - if tmp_path: - obj_path = self.db_path(self.object_path(hexsha)) - obj_dir = dirname(obj_path) - if not isdir(obj_dir): - mkdir(obj_dir) - # END handle destination directory - # rename onto existing doesn't work on windows - if os.name == 'nt' and isfile(obj_path): - remove(obj_path) - # END handle win322 - rename(tmp_path, obj_path) - - # make sure its readable for all ! It started out as rw-- tmp file - # but needs to be rwrr - chmod(obj_path, self.new_objects_mode) - # END handle dry_run - - istream.binsha = hex_to_bin(hexsha) - return istream - - def sha_iter(self): - # find all files which look like an object, extract sha from there - for root, dirs, files in os.walk(self.root_path()): - root_base = basename(root) - if len(root_base) != 2: - continue - - for f in files: - if len(f) != 38: - continue - yield hex_to_bin(root_base + f) - # END for each file - # END for each walk iteration - - def size(self): - return len(tuple(self.sha_iter())) - + """A database which operates on loose object files""" + + # CONFIGURATION + # chunks in which data will be copied between streams + stream_chunk_size = chunk_size + + # On windows we need to keep it writable, otherwise it cannot be removed + # either + new_objects_mode = 0444 + if os.name == 'nt': + new_objects_mode = 0644 + + + def __init__(self, root_path): + super(LooseObjectDB, self).__init__(root_path) + self._hexsha_to_file = dict() + # Additional Flags - might be set to 0 after the first failure + # Depending on the root, this might work for some mounts, for others not, which + # is why it is per instance + self._fd_open_flags = getattr(os, 'O_NOATIME', 0) + + #{ Interface + def object_path(self, hexsha): + """ + :return: path at which the object with the given hexsha would be stored, + relative to the database root""" + return join(hexsha[:2], hexsha[2:]) + + def readable_db_object_path(self, hexsha): + """ + :return: readable object path to the object identified by hexsha + :raise BadObject: If the object file does not exist""" + try: + return self._hexsha_to_file[hexsha] + except KeyError: + pass + # END ignore cache misses + + # try filesystem + path = self.db_path(self.object_path(hexsha)) + if exists(path): + self._hexsha_to_file[hexsha] = path + return path + # END handle cache + raise BadObject(hexsha) + + def partial_to_complete_sha_hex(self, partial_hexsha): + """:return: 20 byte binary sha1 string which matches the given name uniquely + :param name: hexadecimal partial name + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for binsha in self.sha_iter(): + if bin_to_hex(binsha).startswith(partial_hexsha): + # it can't ever find the same object twice + if candidate is not None: + raise AmbiguousObjectName(partial_hexsha) + candidate = binsha + # END for each object + if candidate is None: + raise BadObject(partial_hexsha) + return candidate + + #} END interface + + def _map_loose_object(self, sha): + """ + :return: memory map of that file to allow random read access + :raise BadObject: if object could not be located""" + db_path = self.db_path(self.object_path(bin_to_hex(sha))) + try: + return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) + except OSError,e: + if e.errno != ENOENT: + # try again without noatime + try: + return file_contents_ro_filepath(db_path) + except OSError: + raise BadObject(sha) + # didn't work because of our flag, don't try it again + self._fd_open_flags = 0 + else: + raise BadObject(sha) + # END handle error + # END exception handling + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + finally: + os.close(fd) + # END assure file is closed + + def set_ostream(self, stream): + """:raise TypeError: if the stream does not support the Sha1Writer interface""" + if stream is not None and not isinstance(stream, Sha1Writer): + raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) + return super(LooseObjectDB, self).set_ostream(stream) + + def info(self, sha): + m = self._map_loose_object(sha) + try: + type, size = loose_object_header_info(m) + return OInfo(sha, type, size) + finally: + m.close() + # END assure release of system resources + + def stream(self, sha): + m = self._map_loose_object(sha) + type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) + return OStream(sha, type, size, stream) + + def has_object(self, sha): + try: + self.readable_db_object_path(bin_to_hex(sha)) + return True + except BadObject: + return False + # END check existance + + def store(self, istream): + """note: The sha we produce will be hex by nature""" + tmp_path = None + writer = self.ostream() + if writer is None: + # open a tmp file to write the data to + fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) + + if istream.binsha is None: + writer = FDCompressedSha1Writer(fd) + else: + writer = FDStream(fd) + # END handle direct stream copies + # END handle custom writer + + try: + try: + if istream.binsha is not None: + # copy as much as possible, the actual uncompressed item size might + # be smaller than the compressed version + stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) + else: + # write object with header, we have to make a new one + write_object(istream.type, istream.size, istream.read, writer.write, + chunk_size=self.stream_chunk_size) + # END handle direct stream copies + finally: + if tmp_path: + writer.close() + # END assure target stream is closed + except: + if tmp_path: + os.remove(tmp_path) + raise + # END assure tmpfile removal on error + + hexsha = None + if istream.binsha: + hexsha = istream.hexsha + else: + hexsha = writer.sha(as_hex=True) + # END handle sha + + if tmp_path: + obj_path = self.db_path(self.object_path(hexsha)) + obj_dir = dirname(obj_path) + if not isdir(obj_dir): + mkdir(obj_dir) + # END handle destination directory + # rename onto existing doesn't work on windows + if os.name == 'nt' and isfile(obj_path): + remove(obj_path) + # END handle win322 + rename(tmp_path, obj_path) + + # make sure its readable for all ! It started out as rw-- tmp file + # but needs to be rwrr + chmod(obj_path, self.new_objects_mode) + # END handle dry_run + + istream.binsha = hex_to_bin(hexsha) + return istream + + def sha_iter(self): + # find all files which look like an object, extract sha from there + for root, dirs, files in os.walk(self.root_path()): + root_base = basename(root) + if len(root_base) != 2: + continue + + for f in files: + if len(f) != 38: + continue + yield hex_to_bin(root_base + f) + # END for each file + # END for each walk iteration + + def size(self): + return len(tuple(self.sha_iter())) + diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 5d76c83cc..b9b2b8995 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -5,109 +5,109 @@ """Contains the MemoryDatabase implementation""" from loose import LooseObjectDB from base import ( - ObjectDBR, - ObjectDBW - ) + ObjectDBR, + ObjectDBW + ) from gitdb.base import ( - OStream, - IStream, - ) + OStream, + IStream, + ) from gitdb.exc import ( - BadObject, - UnsupportedOperation - ) + BadObject, + UnsupportedOperation + ) from gitdb.stream import ( - ZippedStoreShaWriter, - DecompressMemMapReader, - ) + ZippedStoreShaWriter, + DecompressMemMapReader, + ) from cStringIO import StringIO __all__ = ("MemoryDB", ) class MemoryDB(ObjectDBR, ObjectDBW): - """A memory database stores everything to memory, providing fast IO and object - retrieval. It should be used to buffer results and obtain SHAs before writing - it to the actual physical storage, as it allows to query whether object already - exists in the target storage before introducing actual IO - - **Note:** memory is currently not threadsafe, hence the async methods cannot be used - for storing""" - - def __init__(self): - super(MemoryDB, self).__init__() - self._db = LooseObjectDB("path/doesnt/matter") - - # maps 20 byte shas to their OStream objects - self._cache = dict() - - def set_ostream(self, stream): - raise UnsupportedOperation("MemoryDB's always stream into memory") - - def store(self, istream): - zstream = ZippedStoreShaWriter() - self._db.set_ostream(zstream) - - istream = self._db.store(istream) - zstream.close() # close to flush - zstream.seek(0) - - # don't provide a size, the stream is written in object format, hence the - # header needs decompression - decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) - self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) - - return istream - - def store_async(self, reader): - raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") - - def has_object(self, sha): - return sha in self._cache + """A memory database stores everything to memory, providing fast IO and object + retrieval. It should be used to buffer results and obtain SHAs before writing + it to the actual physical storage, as it allows to query whether object already + exists in the target storage before introducing actual IO + + **Note:** memory is currently not threadsafe, hence the async methods cannot be used + for storing""" + + def __init__(self): + super(MemoryDB, self).__init__() + self._db = LooseObjectDB("path/doesnt/matter") + + # maps 20 byte shas to their OStream objects + self._cache = dict() + + def set_ostream(self, stream): + raise UnsupportedOperation("MemoryDB's always stream into memory") + + def store(self, istream): + zstream = ZippedStoreShaWriter() + self._db.set_ostream(zstream) + + istream = self._db.store(istream) + zstream.close() # close to flush + zstream.seek(0) + + # don't provide a size, the stream is written in object format, hence the + # header needs decompression + decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) + self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) + + return istream + + def store_async(self, reader): + raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") + + def has_object(self, sha): + return sha in self._cache - def info(self, sha): - # we always return streams, which are infos as well - return self.stream(sha) - - def stream(self, sha): - try: - ostream = self._cache[sha] - # rewind stream for the next one to read - ostream.stream.seek(0) - return ostream - except KeyError: - raise BadObject(sha) - # END exception handling - - def size(self): - return len(self._cache) - - def sha_iter(self): - return self._cache.iterkeys() - - - #{ Interface - def stream_copy(self, sha_iter, odb): - """Copy the streams as identified by sha's yielded by sha_iter into the given odb - The streams will be copied directly - **Note:** the object will only be written if it did not exist in the target db - :return: amount of streams actually copied into odb. If smaller than the amount - of input shas, one or more objects did already exist in odb""" - count = 0 - for sha in sha_iter: - if odb.has_object(sha): - continue - # END check object existance - - ostream = self.stream(sha) - # compressed data including header - sio = StringIO(ostream.stream.data()) - istream = IStream(ostream.type, ostream.size, sio, sha) - - odb.store(istream) - count += 1 - # END for each sha - return count - #} END interface + def info(self, sha): + # we always return streams, which are infos as well + return self.stream(sha) + + def stream(self, sha): + try: + ostream = self._cache[sha] + # rewind stream for the next one to read + ostream.stream.seek(0) + return ostream + except KeyError: + raise BadObject(sha) + # END exception handling + + def size(self): + return len(self._cache) + + def sha_iter(self): + return self._cache.iterkeys() + + + #{ Interface + def stream_copy(self, sha_iter, odb): + """Copy the streams as identified by sha's yielded by sha_iter into the given odb + The streams will be copied directly + **Note:** the object will only be written if it did not exist in the target db + :return: amount of streams actually copied into odb. If smaller than the amount + of input shas, one or more objects did already exist in odb""" + count = 0 + for sha in sha_iter: + if odb.has_object(sha): + continue + # END check object existance + + ostream = self.stream(sha) + # compressed data including header + sio = StringIO(ostream.stream.data()) + istream = IStream(ostream.type, ostream.size, sio, sha) + + odb.store(istream) + count += 1 + # END for each sha + return count + #} END interface diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 4c9d0b919..928731937 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -4,18 +4,18 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing a database to deal with packs""" from base import ( - FileDBBase, - ObjectDBR, - CachingDB - ) + FileDBBase, + ObjectDBR, + CachingDB + ) from gitdb.util import LazyMixin from gitdb.exc import ( - BadObject, - UnsupportedOperation, - AmbiguousObjectName - ) + BadObject, + UnsupportedOperation, + AmbiguousObjectName + ) from gitdb.pack import PackEntity @@ -28,182 +28,182 @@ class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): - """A database operating on a set of object packs""" - - # sort the priority list every N queries - # Higher values are better, performance tests don't show this has - # any effect, but it should have one - _sort_interval = 500 - - def __init__(self, root_path): - super(PackedDB, self).__init__(root_path) - # list of lists with three items: - # * hits - number of times the pack was hit with a request - # * entity - Pack entity instance - # * sha_to_index - PackIndexFile.sha_to_index method for direct cache query - # self._entities = list() # lazy loaded list - self._hit_count = 0 # amount of hits - self._st_mtime = 0 # last modification data of our root path - - def _set_cache_(self, attr): - if attr == '_entities': - self._entities = list() - self.update_cache(force=True) - # END handle entities initialization - - def _sort_entities(self): - self._entities.sort(key=lambda l: l[0], reverse=True) - - def _pack_info(self, sha): - """:return: tuple(entity, index) for an item at the given sha - :param sha: 20 or 40 byte sha - :raise BadObject: - **Note:** This method is not thread-safe, but may be hit in multi-threaded - operation. The worst thing that can happen though is a counter that - was not incremented, or the list being in wrong order. So we safe - the time for locking here, lets see how that goes""" - # presort ? - if self._hit_count % self._sort_interval == 0: - self._sort_entities() - # END update sorting - - for item in self._entities: - index = item[2](sha) - if index is not None: - item[0] += 1 # one hit for you - self._hit_count += 1 # general hit count - return (item[1], index) - # END index found in pack - # END for each item - - # no hit, see whether we have to update packs - # NOTE: considering packs don't change very often, we safe this call - # and leave it to the super-caller to trigger that - raise BadObject(sha) - - #{ Object DB Read - - def has_object(self, sha): - try: - self._pack_info(sha) - return True - except BadObject: - return False - # END exception handling - - def info(self, sha): - entity, index = self._pack_info(sha) - return entity.info_at_index(index) - - def stream(self, sha): - entity, index = self._pack_info(sha) - return entity.stream_at_index(index) - - def sha_iter(self): - sha_list = list() - for entity in self.entities(): - index = entity.index() - sha_by_index = index.sha - for index in xrange(index.size()): - yield sha_by_index(index) - # END for each index - # END for each entity - - def size(self): - sizes = [item[1].index().size() for item in self._entities] - return reduce(lambda x,y: x+y, sizes, 0) - - #} END object db read - - #{ object db write - - def store(self, istream): - """Storing individual objects is not feasible as a pack is designed to - hold multiple objects. Writing or rewriting packs for single objects is - inefficient""" - raise UnsupportedOperation() - - def store_async(self, reader): - # TODO: add ObjectDBRW before implementing this - raise NotImplementedError() - - #} END object db write - - - #{ Interface - - def update_cache(self, force=False): - """ - Update our cache with the acutally existing packs on disk. Add new ones, - and remove deleted ones. We keep the unchanged ones - - :param force: If True, the cache will be updated even though the directory - does not appear to have changed according to its modification timestamp. - :return: True if the packs have been updated so there is new information, - False if there was no change to the pack database""" - stat = os.stat(self.root_path()) - if not force and stat.st_mtime <= self._st_mtime: - return False - # END abort early on no change - self._st_mtime = stat.st_mtime - - # packs are supposed to be prefixed with pack- by git-convention - # get all pack files, figure out what changed - pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) - our_pack_files = set(item[1].pack().path() for item in self._entities) - - # new packs - for pack_file in (pack_files - our_pack_files): - # init the hit-counter/priority with the size, a good measure for hit- - # probability. Its implemented so that only 12 bytes will be read - entity = PackEntity(pack_file) - self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) - # END for each new packfile - - # removed packs - for pack_file in (our_pack_files - pack_files): - del_index = -1 - for i, item in enumerate(self._entities): - if item[1].pack().path() == pack_file: - del_index = i - break - # END found index - # END for each entity - assert del_index != -1 - del(self._entities[del_index]) - # END for each removed pack - - # reinitialize prioritiess - self._sort_entities() - return True - - def entities(self): - """:return: list of pack entities operated upon by this database""" - return [ item[1] for item in self._entities ] - - def partial_to_complete_sha(self, partial_binsha, canonical_length): - """:return: 20 byte sha as inferred by the given partial binary sha - :param partial_binsha: binary sha with less than 20 bytes - :param canonical_length: length of the corresponding canonical representation. - It is required as binary sha's cannot display whether the original hex sha - had an odd or even number of characters - :raise AmbiguousObjectName: - :raise BadObject: """ - candidate = None - for item in self._entities: - item_index = item[1].index().partial_sha_to_index(partial_binsha, canonical_length) - if item_index is not None: - sha = item[1].index().sha(item_index) - if candidate and candidate != sha: - raise AmbiguousObjectName(partial_binsha) - candidate = sha - # END handle full sha could be found - # END for each entity - - if candidate: - return candidate - - # still not found ? - raise BadObject(partial_binsha) - - #} END interface + """A database operating on a set of object packs""" + + # sort the priority list every N queries + # Higher values are better, performance tests don't show this has + # any effect, but it should have one + _sort_interval = 500 + + def __init__(self, root_path): + super(PackedDB, self).__init__(root_path) + # list of lists with three items: + # * hits - number of times the pack was hit with a request + # * entity - Pack entity instance + # * sha_to_index - PackIndexFile.sha_to_index method for direct cache query + # self._entities = list() # lazy loaded list + self._hit_count = 0 # amount of hits + self._st_mtime = 0 # last modification data of our root path + + def _set_cache_(self, attr): + if attr == '_entities': + self._entities = list() + self.update_cache(force=True) + # END handle entities initialization + + def _sort_entities(self): + self._entities.sort(key=lambda l: l[0], reverse=True) + + def _pack_info(self, sha): + """:return: tuple(entity, index) for an item at the given sha + :param sha: 20 or 40 byte sha + :raise BadObject: + **Note:** This method is not thread-safe, but may be hit in multi-threaded + operation. The worst thing that can happen though is a counter that + was not incremented, or the list being in wrong order. So we safe + the time for locking here, lets see how that goes""" + # presort ? + if self._hit_count % self._sort_interval == 0: + self._sort_entities() + # END update sorting + + for item in self._entities: + index = item[2](sha) + if index is not None: + item[0] += 1 # one hit for you + self._hit_count += 1 # general hit count + return (item[1], index) + # END index found in pack + # END for each item + + # no hit, see whether we have to update packs + # NOTE: considering packs don't change very often, we safe this call + # and leave it to the super-caller to trigger that + raise BadObject(sha) + + #{ Object DB Read + + def has_object(self, sha): + try: + self._pack_info(sha) + return True + except BadObject: + return False + # END exception handling + + def info(self, sha): + entity, index = self._pack_info(sha) + return entity.info_at_index(index) + + def stream(self, sha): + entity, index = self._pack_info(sha) + return entity.stream_at_index(index) + + def sha_iter(self): + sha_list = list() + for entity in self.entities(): + index = entity.index() + sha_by_index = index.sha + for index in xrange(index.size()): + yield sha_by_index(index) + # END for each index + # END for each entity + + def size(self): + sizes = [item[1].index().size() for item in self._entities] + return reduce(lambda x,y: x+y, sizes, 0) + + #} END object db read + + #{ object db write + + def store(self, istream): + """Storing individual objects is not feasible as a pack is designed to + hold multiple objects. Writing or rewriting packs for single objects is + inefficient""" + raise UnsupportedOperation() + + def store_async(self, reader): + # TODO: add ObjectDBRW before implementing this + raise NotImplementedError() + + #} END object db write + + + #{ Interface + + def update_cache(self, force=False): + """ + Update our cache with the acutally existing packs on disk. Add new ones, + and remove deleted ones. We keep the unchanged ones + + :param force: If True, the cache will be updated even though the directory + does not appear to have changed according to its modification timestamp. + :return: True if the packs have been updated so there is new information, + False if there was no change to the pack database""" + stat = os.stat(self.root_path()) + if not force and stat.st_mtime <= self._st_mtime: + return False + # END abort early on no change + self._st_mtime = stat.st_mtime + + # packs are supposed to be prefixed with pack- by git-convention + # get all pack files, figure out what changed + pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) + our_pack_files = set(item[1].pack().path() for item in self._entities) + + # new packs + for pack_file in (pack_files - our_pack_files): + # init the hit-counter/priority with the size, a good measure for hit- + # probability. Its implemented so that only 12 bytes will be read + entity = PackEntity(pack_file) + self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) + # END for each new packfile + + # removed packs + for pack_file in (our_pack_files - pack_files): + del_index = -1 + for i, item in enumerate(self._entities): + if item[1].pack().path() == pack_file: + del_index = i + break + # END found index + # END for each entity + assert del_index != -1 + del(self._entities[del_index]) + # END for each removed pack + + # reinitialize prioritiess + self._sort_entities() + return True + + def entities(self): + """:return: list of pack entities operated upon by this database""" + return [ item[1] for item in self._entities ] + + def partial_to_complete_sha(self, partial_binsha, canonical_length): + """:return: 20 byte sha as inferred by the given partial binary sha + :param partial_binsha: binary sha with less than 20 bytes + :param canonical_length: length of the corresponding canonical representation. + It is required as binary sha's cannot display whether the original hex sha + had an odd or even number of characters + :raise AmbiguousObjectName: + :raise BadObject: """ + candidate = None + for item in self._entities: + item_index = item[1].index().partial_sha_to_index(partial_binsha, canonical_length) + if item_index is not None: + sha = item[1].index().sha(item_index) + if candidate and candidate != sha: + raise AmbiguousObjectName(partial_binsha) + candidate = sha + # END handle full sha could be found + # END for each entity + + if candidate: + return candidate + + # still not found ? + raise BadObject(partial_binsha) + + #} END interface diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 898984323..60004a77a 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -3,77 +3,77 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from base import ( - CompoundDB, - ) + CompoundDB, + ) import os __all__ = ('ReferenceDB', ) class ReferenceDB(CompoundDB): - """A database consisting of database referred to in a file""" - - # Configuration - # Specifies the object database to use for the paths found in the alternates - # file. If None, it defaults to the GitDB - ObjectDBCls = None - - def __init__(self, ref_file): - super(ReferenceDB, self).__init__() - self._ref_file = ref_file - - def _set_cache_(self, attr): - if attr == '_dbs': - self._dbs = list() - self._update_dbs_from_ref_file() - else: - super(ReferenceDB, self)._set_cache_(attr) - # END handle attrs - - def _update_dbs_from_ref_file(self): - dbcls = self.ObjectDBCls - if dbcls is None: - # late import - from git import GitDB - dbcls = GitDB - # END get db type - - # try to get as many as possible, don't fail if some are unavailable - ref_paths = list() - try: - ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] - except (OSError, IOError): - pass - # END handle alternates - - ref_paths_set = set(ref_paths) - cur_ref_paths_set = set(db.root_path() for db in self._dbs) - - # remove existing - for path in (cur_ref_paths_set - ref_paths_set): - for i, db in enumerate(self._dbs[:]): - if db.root_path() == path: - del(self._dbs[i]) - continue - # END del matching db - # END for each path to remove - - # add new - # sort them to maintain order - added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) - for path in added_paths: - try: - db = dbcls(path) - # force an update to verify path - if isinstance(db, CompoundDB): - db.databases() - # END verification - self._dbs.append(db) - except Exception, e: - # ignore invalid paths or issues - pass - # END for each path to add - - def update_cache(self, force=False): - # re-read alternates and update databases - self._update_dbs_from_ref_file() - return super(ReferenceDB, self).update_cache(force) + """A database consisting of database referred to in a file""" + + # Configuration + # Specifies the object database to use for the paths found in the alternates + # file. If None, it defaults to the GitDB + ObjectDBCls = None + + def __init__(self, ref_file): + super(ReferenceDB, self).__init__() + self._ref_file = ref_file + + def _set_cache_(self, attr): + if attr == '_dbs': + self._dbs = list() + self._update_dbs_from_ref_file() + else: + super(ReferenceDB, self)._set_cache_(attr) + # END handle attrs + + def _update_dbs_from_ref_file(self): + dbcls = self.ObjectDBCls + if dbcls is None: + # late import + from git import GitDB + dbcls = GitDB + # END get db type + + # try to get as many as possible, don't fail if some are unavailable + ref_paths = list() + try: + ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] + except (OSError, IOError): + pass + # END handle alternates + + ref_paths_set = set(ref_paths) + cur_ref_paths_set = set(db.root_path() for db in self._dbs) + + # remove existing + for path in (cur_ref_paths_set - ref_paths_set): + for i, db in enumerate(self._dbs[:]): + if db.root_path() == path: + del(self._dbs[i]) + continue + # END del matching db + # END for each path to remove + + # add new + # sort them to maintain order + added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) + for path in added_paths: + try: + db = dbcls(path) + # force an update to verify path + if isinstance(db, CompoundDB): + db.databases() + # END verification + self._dbs.append(db) + except Exception, e: + # ignore invalid paths or issues + pass + # END for each path to add + + def update_cache(self, force=False): + # re-read alternates and update databases + self._update_dbs_from_ref_file() + return super(ReferenceDB, self).update_cache(force) diff --git a/gitdb/exc.py b/gitdb/exc.py index e087047b4..7180fb586 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -6,27 +6,27 @@ from util import to_hex_sha class ODBError(Exception): - """All errors thrown by the object database""" - + """All errors thrown by the object database""" + class InvalidDBRoot(ODBError): - """Thrown if an object database cannot be initialized at the given path""" - + """Thrown if an object database cannot be initialized at the given path""" + class BadObject(ODBError): - """The object with the given SHA does not exist. Instantiate with the - failed sha""" - - def __str__(self): - return "BadObject: %s" % to_hex_sha(self.args[0]) - + """The object with the given SHA does not exist. Instantiate with the + failed sha""" + + def __str__(self): + return "BadObject: %s" % to_hex_sha(self.args[0]) + class ParseError(ODBError): - """Thrown if the parsing of a file failed due to an invalid format""" + """Thrown if the parsing of a file failed due to an invalid format""" class AmbiguousObjectName(ODBError): - """Thrown if a possibly shortened name does not uniquely represent a single object - in the database""" + """Thrown if a possibly shortened name does not uniquely represent a single object + in the database""" class BadObjectType(ODBError): - """The object had an unsupported type""" + """The object had an unsupported type""" class UnsupportedOperation(ODBError): - """Thrown if the given operation cannot be supported by the object database""" + """Thrown if the given operation cannot be supported by the object database""" diff --git a/gitdb/ext/async b/gitdb/ext/async index 039c1d5c2..571412931 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 039c1d5c26bc2ceaa9e55082efae2068d9873e45 +Subproject commit 571412931829200aff06a44b9c5524e122e524e9 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 360a8956f..1b3ab5598 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 360a8956fe73a0a96315e946f52737569d990369 +Subproject commit 1b3ab5598e93369282502d049d64cb2ca12839cb diff --git a/gitdb/fun.py b/gitdb/fun.py index 66130ebee..c1e73e895 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -7,8 +7,8 @@ it into c later, if required""" from exc import ( - BadObjectType - ) + BadObjectType + ) from util import zlib decompressobj = zlib.decompressobj @@ -23,655 +23,655 @@ REF_DELTA = 7 delta_types = (OFS_DELTA, REF_DELTA) -type_id_to_type_map = { - 0 : "", # EXT 1 - 1 : "commit", - 2 : "tree", - 3 : "blob", - 4 : "tag", - 5 : "", # EXT 2 - OFS_DELTA : "OFS_DELTA", # OFFSET DELTA - REF_DELTA : "REF_DELTA" # REFERENCE DELTA - } +type_id_to_type_map = { + 0 : "", # EXT 1 + 1 : "commit", + 2 : "tree", + 3 : "blob", + 4 : "tag", + 5 : "", # EXT 2 + OFS_DELTA : "OFS_DELTA", # OFFSET DELTA + REF_DELTA : "REF_DELTA" # REFERENCE DELTA + } type_to_type_id_map = dict( - commit=1, - tree=2, - blob=3, - tag=4, - OFS_DELTA=OFS_DELTA, - REF_DELTA=REF_DELTA - ) + commit=1, + tree=2, + blob=3, + tag=4, + OFS_DELTA=OFS_DELTA, + REF_DELTA=REF_DELTA + ) # used when dealing with larger streams chunk_size = 1000*mmap.PAGESIZE __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', - 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', + 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') #{ Structures def _set_delta_rbound(d, size): - """Truncate the given delta to the given size - :param size: size relative to our target offset, may not be 0, must be smaller or equal - to our size - :return: d""" - d.ts = size - - # NOTE: data is truncated automatically when applying the delta - # MUST NOT DO THIS HERE - return d - + """Truncate the given delta to the given size + :param size: size relative to our target offset, may not be 0, must be smaller or equal + to our size + :return: d""" + d.ts = size + + # NOTE: data is truncated automatically when applying the delta + # MUST NOT DO THIS HERE + return d + def _move_delta_lbound(d, bytes): - """Move the delta by the given amount of bytes, reducing its size so that its - right bound stays static - :param bytes: amount of bytes to move, must be smaller than delta size - :return: d""" - if bytes == 0: - return - - d.to += bytes - d.so += bytes - d.ts -= bytes - if d.data is not None: - d.data = d.data[bytes:] - # END handle data - - return d - + """Move the delta by the given amount of bytes, reducing its size so that its + right bound stays static + :param bytes: amount of bytes to move, must be smaller than delta size + :return: d""" + if bytes == 0: + return + + d.to += bytes + d.so += bytes + d.ts -= bytes + if d.data is not None: + d.data = d.data[bytes:] + # END handle data + + return d + def delta_duplicate(src): - return DeltaChunk(src.to, src.ts, src.so, src.data) - + return DeltaChunk(src.to, src.ts, src.so, src.data) + def delta_chunk_apply(dc, bbuf, write): - """Apply own data to the target buffer - :param bbuf: buffer providing source bytes for copy operations - :param write: write method to call with data to write""" - if dc.data is None: - # COPY DATA FROM SOURCE - write(buffer(bbuf, dc.so, dc.ts)) - else: - # APPEND DATA - # whats faster: if + 4 function calls or just a write with a slice ? - # Considering data can be larger than 127 bytes now, it should be worth it - if dc.ts < len(dc.data): - write(dc.data[:dc.ts]) - else: - write(dc.data) - # END handle truncation - # END handle chunk mode + """Apply own data to the target buffer + :param bbuf: buffer providing source bytes for copy operations + :param write: write method to call with data to write""" + if dc.data is None: + # COPY DATA FROM SOURCE + write(buffer(bbuf, dc.so, dc.ts)) + else: + # APPEND DATA + # whats faster: if + 4 function calls or just a write with a slice ? + # Considering data can be larger than 127 bytes now, it should be worth it + if dc.ts < len(dc.data): + write(dc.data[:dc.ts]) + else: + write(dc.data) + # END handle truncation + # END handle chunk mode class DeltaChunk(object): - """Represents a piece of a delta, it can either add new data, or copy existing - one from a source buffer""" - __slots__ = ( - 'to', # start offset in the target buffer in bytes - 'ts', # size of this chunk in the target buffer in bytes - 'so', # start offset in the source buffer in bytes or None - 'data', # chunk of bytes to be added to the target buffer, - # DeltaChunkList to use as base, or None - ) - - def __init__(self, to, ts, so, data): - self.to = to - self.ts = ts - self.so = so - self.data = data + """Represents a piece of a delta, it can either add new data, or copy existing + one from a source buffer""" + __slots__ = ( + 'to', # start offset in the target buffer in bytes + 'ts', # size of this chunk in the target buffer in bytes + 'so', # start offset in the source buffer in bytes or None + 'data', # chunk of bytes to be added to the target buffer, + # DeltaChunkList to use as base, or None + ) + + def __init__(self, to, ts, so, data): + self.to = to + self.ts = ts + self.so = so + self.data = data - def __repr__(self): - return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") - - #{ Interface - - def rbound(self): - return self.to + self.ts - - def has_data(self): - """:return: True if the instance has data to add to the target stream""" - return self.data is not None - - #} END interface + def __repr__(self): + return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") + + #{ Interface + + def rbound(self): + return self.to + self.ts + + def has_data(self): + """:return: True if the instance has data to add to the target stream""" + return self.data is not None + + #} END interface def _closest_index(dcl, absofs): - """:return: index at which the given absofs should be inserted. The index points - to the DeltaChunk with a target buffer absofs that equals or is greater than - absofs. - **Note:** global method for performance only, it belongs to DeltaChunkList""" - lo = 0 - hi = len(dcl) - while lo < hi: - mid = (lo + hi) / 2 - dc = dcl[mid] - if dc.to > absofs: - hi = mid - elif dc.rbound() > absofs or dc.to == absofs: - return mid - else: - lo = mid + 1 - # END handle bound - # END for each delta absofs - return len(dcl)-1 - + """:return: index at which the given absofs should be inserted. The index points + to the DeltaChunk with a target buffer absofs that equals or is greater than + absofs. + **Note:** global method for performance only, it belongs to DeltaChunkList""" + lo = 0 + hi = len(dcl) + while lo < hi: + mid = (lo + hi) / 2 + dc = dcl[mid] + if dc.to > absofs: + hi = mid + elif dc.rbound() > absofs or dc.to == absofs: + return mid + else: + lo = mid + 1 + # END handle bound + # END for each delta absofs + return len(dcl)-1 + def delta_list_apply(dcl, bbuf, write): - """Apply the chain's changes and write the final result using the passed - write function. - :param bbuf: base buffer containing the base of all deltas contained in this - list. It will only be used if the chunk in question does not have a base - chain. - :param write: function taking a string of bytes to write to the output""" - for dc in dcl: - delta_chunk_apply(dc, bbuf, write) - # END for each dc + """Apply the chain's changes and write the final result using the passed + write function. + :param bbuf: base buffer containing the base of all deltas contained in this + list. It will only be used if the chunk in question does not have a base + chain. + :param write: function taking a string of bytes to write to the output""" + for dc in dcl: + delta_chunk_apply(dc, bbuf, write) + # END for each dc def delta_list_slice(dcl, absofs, size, ndcl): - """:return: Subsection of this list at the given absolute offset, with the given - size in bytes. - :return: None""" - cdi = _closest_index(dcl, absofs) # delta start index - cd = dcl[cdi] - slen = len(dcl) - lappend = ndcl.append - - if cd.to != absofs: - tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) - _move_delta_lbound(tcd, absofs - cd.to) - tcd.ts = min(tcd.ts, size) - lappend(tcd) - size -= tcd.ts - cdi += 1 - # END lbound overlap handling - - while cdi < slen and size: - # are we larger than the current block - cd = dcl[cdi] - if cd.ts <= size: - lappend(DeltaChunk(cd.to, cd.ts, cd.so, cd.data)) - size -= cd.ts - else: - tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) - tcd.ts = size - lappend(tcd) - size -= tcd.ts - break - # END hadle size - cdi += 1 - # END for each chunk - - + """:return: Subsection of this list at the given absolute offset, with the given + size in bytes. + :return: None""" + cdi = _closest_index(dcl, absofs) # delta start index + cd = dcl[cdi] + slen = len(dcl) + lappend = ndcl.append + + if cd.to != absofs: + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) + _move_delta_lbound(tcd, absofs - cd.to) + tcd.ts = min(tcd.ts, size) + lappend(tcd) + size -= tcd.ts + cdi += 1 + # END lbound overlap handling + + while cdi < slen and size: + # are we larger than the current block + cd = dcl[cdi] + if cd.ts <= size: + lappend(DeltaChunk(cd.to, cd.ts, cd.so, cd.data)) + size -= cd.ts + else: + tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) + tcd.ts = size + lappend(tcd) + size -= tcd.ts + break + # END hadle size + cdi += 1 + # END for each chunk + + class DeltaChunkList(list): - """List with special functionality to deal with DeltaChunks. - There are two types of lists we represent. The one was created bottom-up, working - towards the latest delta, the other kind was created top-down, working from the - latest delta down to the earliest ancestor. This attribute is queryable - after all processing with is_reversed.""" - - __slots__ = tuple() - - def rbound(self): - """:return: rightmost extend in bytes, absolute""" - if len(self) == 0: - return 0 - return self[-1].rbound() - - def lbound(self): - """:return: leftmost byte at which this chunklist starts""" - if len(self) == 0: - return 0 - return self[0].to - - def size(self): - """:return: size of bytes as measured by our delta chunks""" - return self.rbound() - self.lbound() - - def apply(self, bbuf, write): - """Only used by public clients, internally we only use the global routines - for performance""" - return delta_list_apply(self, bbuf, write) - - def compress(self): - """Alter the list to reduce the amount of nodes. Currently we concatenate - add-chunks - :return: self""" - slen = len(self) - if slen < 2: - return self - i = 0 - slen_orig = slen - - first_data_index = None - while i < slen: - dc = self[i] - i += 1 - if dc.data is None: - if first_data_index is not None and i-2-first_data_index > 1: - #if first_data_index is not None: - nd = StringIO() # new data - so = self[first_data_index].to # start offset in target buffer - for x in xrange(first_data_index, i-1): - xdc = self[x] - nd.write(xdc.data[:xdc.ts]) - # END collect data - - del(self[first_data_index:i-1]) - buf = nd.getvalue() - self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) - - slen = len(self) - i = first_data_index + 1 - - # END concatenate data - first_data_index = None - continue - # END skip non-data chunks - - if first_data_index is None: - first_data_index = i-1 - # END iterate list - - #if slen_orig != len(self): - # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) - return self - - def check_integrity(self, target_size=-1): - """Verify the list has non-overlapping chunks only, and the total size matches - target_size - :param target_size: if not -1, the total size of the chain must be target_size - :raise AssertionError: if the size doen't match""" - if target_size > -1: - assert self[-1].rbound() == target_size - assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size - # END target size verification - - if len(self) < 2: - return - - # check data - for dc in self: - assert dc.ts > 0 - if dc.has_data(): - assert len(dc.data) >= dc.ts - # END for each dc - - left = islice(self, 0, len(self)-1) - right = iter(self) - right.next() - # this is very pythonic - we might have just use index based access here, - # but this could actually be faster - for lft,rgt in izip(left, right): - assert lft.rbound() == rgt.to - assert lft.to + lft.ts == rgt.to - # END for each pair - + """List with special functionality to deal with DeltaChunks. + There are two types of lists we represent. The one was created bottom-up, working + towards the latest delta, the other kind was created top-down, working from the + latest delta down to the earliest ancestor. This attribute is queryable + after all processing with is_reversed.""" + + __slots__ = tuple() + + def rbound(self): + """:return: rightmost extend in bytes, absolute""" + if len(self) == 0: + return 0 + return self[-1].rbound() + + def lbound(self): + """:return: leftmost byte at which this chunklist starts""" + if len(self) == 0: + return 0 + return self[0].to + + def size(self): + """:return: size of bytes as measured by our delta chunks""" + return self.rbound() - self.lbound() + + def apply(self, bbuf, write): + """Only used by public clients, internally we only use the global routines + for performance""" + return delta_list_apply(self, bbuf, write) + + def compress(self): + """Alter the list to reduce the amount of nodes. Currently we concatenate + add-chunks + :return: self""" + slen = len(self) + if slen < 2: + return self + i = 0 + slen_orig = slen + + first_data_index = None + while i < slen: + dc = self[i] + i += 1 + if dc.data is None: + if first_data_index is not None and i-2-first_data_index > 1: + #if first_data_index is not None: + nd = StringIO() # new data + so = self[first_data_index].to # start offset in target buffer + for x in xrange(first_data_index, i-1): + xdc = self[x] + nd.write(xdc.data[:xdc.ts]) + # END collect data + + del(self[first_data_index:i-1]) + buf = nd.getvalue() + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + + slen = len(self) + i = first_data_index + 1 + + # END concatenate data + first_data_index = None + continue + # END skip non-data chunks + + if first_data_index is None: + first_data_index = i-1 + # END iterate list + + #if slen_orig != len(self): + # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) + return self + + def check_integrity(self, target_size=-1): + """Verify the list has non-overlapping chunks only, and the total size matches + target_size + :param target_size: if not -1, the total size of the chain must be target_size + :raise AssertionError: if the size doen't match""" + if target_size > -1: + assert self[-1].rbound() == target_size + assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size + # END target size verification + + if len(self) < 2: + return + + # check data + for dc in self: + assert dc.ts > 0 + if dc.has_data(): + assert len(dc.data) >= dc.ts + # END for each dc + + left = islice(self, 0, len(self)-1) + right = iter(self) + right.next() + # this is very pythonic - we might have just use index based access here, + # but this could actually be faster + for lft,rgt in izip(left, right): + assert lft.rbound() == rgt.to + assert lft.to + lft.ts == rgt.to + # END for each pair + class TopdownDeltaChunkList(DeltaChunkList): - """Represents a list which is generated by feeding its ancestor streams one by - one""" - __slots__ = tuple() - - def connect_with_next_base(self, bdcl): - """Connect this chain with the next level of our base delta chunklist. - The goal in this game is to mark as many of our chunks rigid, hence they - cannot be changed by any of the upcoming bases anymore. Once all our - chunks are marked like that, we can stop all processing - :param bdcl: data chunk list being one of our bases. They must be fed in - consequtively and in order, towards the earliest ancestor delta - :return: True if processing was done. Use it to abort processing of - remaining streams if False is returned""" - nfc = 0 # number of frozen chunks - dci = 0 # delta chunk index - slen = len(self) # len of self - ccl = list() # temporary list - while dci < slen: - dc = self[dci] - dci += 1 - - # all add-chunks which are already topmost don't need additional processing - if dc.data is not None: - nfc += 1 - continue - # END skip add chunks - - # copy chunks - # integrate the portion of the base list into ourselves. Lists - # dont support efficient insertion ( just one at a time ), but for now - # we live with it. Internally, its all just a 32/64bit pointer, and - # the portions of moved memory should be smallish. Maybe we just rebuild - # ourselves in order to reduce the amount of insertions ... - del(ccl[:]) - delta_list_slice(bdcl, dc.so, dc.ts, ccl) - - # move the target bounds into place to match with our chunk - ofs = dc.to - dc.so - for cdc in ccl: - cdc.to += ofs - # END update target bounds - - if len(ccl) == 1: - self[dci-1] = ccl[0] - else: - # maybe try to compute the expenses here, and pick the right algorithm - # It would normally be faster than copying everything physically though - # TODO: Use a deque here, and decide by the index whether to extend - # or extend left ! - post_dci = self[dci:] - del(self[dci-1:]) # include deletion of dc - self.extend(ccl) - self.extend(post_dci) - - slen = len(self) - dci += len(ccl)-1 # deleted dc, added rest - - # END handle chunk replacement - # END for each chunk - - if nfc == slen: - return False - # END handle completeness - return True - - + """Represents a list which is generated by feeding its ancestor streams one by + one""" + __slots__ = tuple() + + def connect_with_next_base(self, bdcl): + """Connect this chain with the next level of our base delta chunklist. + The goal in this game is to mark as many of our chunks rigid, hence they + cannot be changed by any of the upcoming bases anymore. Once all our + chunks are marked like that, we can stop all processing + :param bdcl: data chunk list being one of our bases. They must be fed in + consequtively and in order, towards the earliest ancestor delta + :return: True if processing was done. Use it to abort processing of + remaining streams if False is returned""" + nfc = 0 # number of frozen chunks + dci = 0 # delta chunk index + slen = len(self) # len of self + ccl = list() # temporary list + while dci < slen: + dc = self[dci] + dci += 1 + + # all add-chunks which are already topmost don't need additional processing + if dc.data is not None: + nfc += 1 + continue + # END skip add chunks + + # copy chunks + # integrate the portion of the base list into ourselves. Lists + # dont support efficient insertion ( just one at a time ), but for now + # we live with it. Internally, its all just a 32/64bit pointer, and + # the portions of moved memory should be smallish. Maybe we just rebuild + # ourselves in order to reduce the amount of insertions ... + del(ccl[:]) + delta_list_slice(bdcl, dc.so, dc.ts, ccl) + + # move the target bounds into place to match with our chunk + ofs = dc.to - dc.so + for cdc in ccl: + cdc.to += ofs + # END update target bounds + + if len(ccl) == 1: + self[dci-1] = ccl[0] + else: + # maybe try to compute the expenses here, and pick the right algorithm + # It would normally be faster than copying everything physically though + # TODO: Use a deque here, and decide by the index whether to extend + # or extend left ! + post_dci = self[dci:] + del(self[dci-1:]) # include deletion of dc + self.extend(ccl) + self.extend(post_dci) + + slen = len(self) + dci += len(ccl)-1 # deleted dc, added rest + + # END handle chunk replacement + # END for each chunk + + if nfc == slen: + return False + # END handle completeness + return True + + #} END structures #{ Routines def is_loose_object(m): - """ - :return: True the file contained in memory map m appears to be a loose object. - Only the first two bytes are needed""" - b0, b1 = map(ord, m[:2]) - word = (b0 << 8) + b1 - return b0 == 0x78 and (word % 31) == 0 + """ + :return: True the file contained in memory map m appears to be a loose object. + Only the first two bytes are needed""" + b0, b1 = map(ord, m[:2]) + word = (b0 << 8) + b1 + return b0 == 0x78 and (word % 31) == 0 def loose_object_header_info(m): - """ - :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the - object as well as its uncompressed size in bytes. - :param m: memory map from which to read the compressed object data""" - decompress_size = 8192 # is used in cgit as well - hdr = decompressobj().decompress(m, decompress_size) - type_name, size = hdr[:hdr.find("\0")].split(" ") - return type_name, int(size) - + """ + :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + object as well as its uncompressed size in bytes. + :param m: memory map from which to read the compressed object data""" + decompress_size = 8192 # is used in cgit as well + hdr = decompressobj().decompress(m, decompress_size) + type_name, size = hdr[:hdr.find("\0")].split(" ") + return type_name, int(size) + def pack_object_header_info(data): - """ - :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) - The type_id should be interpreted according to the ``type_id_to_type_map`` map - The byte-offset specifies the start of the actual zlib compressed datastream - :param m: random-access memory, like a string or memory map""" - c = ord(data[0]) # first byte - i = 1 # next char to read - type_id = (c >> 4) & 7 # numeric type - size = c & 15 # starting size - s = 4 # starting bit-shift size - while c & 0x80: - c = ord(data[i]) - i += 1 - size += (c & 0x7f) << s - s += 7 - # END character loop - return (type_id, size, i) + """ + :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) + The type_id should be interpreted according to the ``type_id_to_type_map`` map + The byte-offset specifies the start of the actual zlib compressed datastream + :param m: random-access memory, like a string or memory map""" + c = ord(data[0]) # first byte + i = 1 # next char to read + type_id = (c >> 4) & 7 # numeric type + size = c & 15 # starting size + s = 4 # starting bit-shift size + while c & 0x80: + c = ord(data[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + return (type_id, size, i) def create_pack_object_header(obj_type, obj_size): - """ - :return: string defining the pack header comprised of the object type - and its incompressed size in bytes - - :param obj_type: pack type_id of the object - :param obj_size: uncompressed size in bytes of the following object stream""" - c = 0 # 1 byte - hdr = str() # output string + """ + :return: string defining the pack header comprised of the object type + and its incompressed size in bytes + + :param obj_type: pack type_id of the object + :param obj_size: uncompressed size in bytes of the following object stream""" + c = 0 # 1 byte + hdr = str() # output string - c = (obj_type << 4) | (obj_size & 0xf) - obj_size >>= 4 - while obj_size: - hdr += chr(c | 0x80) - c = obj_size & 0x7f - obj_size >>= 7 - #END until size is consumed - hdr += chr(c) - return hdr - + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr += chr(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + #END until size is consumed + hdr += chr(c) + return hdr + def msb_size(data, offset=0): - """ - :return: tuple(read_bytes, size) read the msb size from the given random - access data starting at the given byte offset""" - size = 0 - i = 0 - l = len(data) - hit_msb = False - while i < l: - c = ord(data[i+offset]) - size |= (c & 0x7f) << i*7 - i += 1 - if not c & 0x80: - hit_msb = True - break - # END check msb bit - # END while in range - if not hit_msb: - raise AssertionError("Could not find terminating MSB byte in data stream") - return i+offset, size - + """ + :return: tuple(read_bytes, size) read the msb size from the given random + access data starting at the given byte offset""" + size = 0 + i = 0 + l = len(data) + hit_msb = False + while i < l: + c = ord(data[i+offset]) + size |= (c & 0x7f) << i*7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + if not hit_msb: + raise AssertionError("Could not find terminating MSB byte in data stream") + return i+offset, size + def loose_object_header(type, size): - """ - :return: string representing the loose object header, which is immediately - followed by the content stream of size 'size'""" - return "%s %i\0" % (type, size) - + """ + :return: string representing the loose object header, which is immediately + followed by the content stream of size 'size'""" + return "%s %i\0" % (type, size) + def write_object(type, size, read, write, chunk_size=chunk_size): - """ - Write the object as identified by type, size and source_stream into the - target_stream - - :param type: type string of the object - :param size: amount of bytes to write from source_stream - :param read: read method of a stream providing the content data - :param write: write method of the output stream - :param close_target_stream: if True, the target stream will be closed when - the routine exits, even if an error is thrown - :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" - tbw = 0 # total num bytes written - - # WRITE HEADER: type SP size NULL - tbw += write(loose_object_header(type, size)) - tbw += stream_copy(read, write, size, chunk_size) - - return tbw + """ + Write the object as identified by type, size and source_stream into the + target_stream + + :param type: type string of the object + :param size: amount of bytes to write from source_stream + :param read: read method of a stream providing the content data + :param write: write method of the output stream + :param close_target_stream: if True, the target stream will be closed when + the routine exits, even if an error is thrown + :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" + tbw = 0 # total num bytes written + + # WRITE HEADER: type SP size NULL + tbw += write(loose_object_header(type, size)) + tbw += stream_copy(read, write, size, chunk_size) + + return tbw def stream_copy(read, write, size, chunk_size): - """ - Copy a stream up to size bytes using the provided read and write methods, - in chunks of chunk_size - - **Note:** its much like stream_copy utility, but operates just using methods""" - dbw = 0 # num data bytes written - - # WRITE ALL DATA UP TO SIZE - while True: - cs = min(chunk_size, size-dbw) - # NOTE: not all write methods return the amount of written bytes, like - # mmap.write. Its bad, but we just deal with it ... perhaps its not - # even less efficient - # data_len = write(read(cs)) - # dbw += data_len - data = read(cs) - data_len = len(data) - dbw += data_len - write(data) - if data_len < cs or dbw == size: - break - # END check for stream end - # END duplicate data - return dbw - + """ + Copy a stream up to size bytes using the provided read and write methods, + in chunks of chunk_size + + **Note:** its much like stream_copy utility, but operates just using methods""" + dbw = 0 # num data bytes written + + # WRITE ALL DATA UP TO SIZE + while True: + cs = min(chunk_size, size-dbw) + # NOTE: not all write methods return the amount of written bytes, like + # mmap.write. Its bad, but we just deal with it ... perhaps its not + # even less efficient + # data_len = write(read(cs)) + # dbw += data_len + data = read(cs) + data_len = len(data) + dbw += data_len + write(data) + if data_len < cs or dbw == size: + break + # END check for stream end + # END duplicate data + return dbw + def connect_deltas(dstreams): - """ - Read the condensed delta chunk information from dstream and merge its information - into a list of existing delta chunks - - :param dstreams: iterable of delta stream objects, the delta to be applied last - comes first, then all its ancestors in order - :return: DeltaChunkList, containing all operations to apply""" - tdcl = None # topmost dcl - - dcl = tdcl = TopdownDeltaChunkList() - for dsi, ds in enumerate(dstreams): - # print "Stream", dsi - db = ds.read() - delta_buf_size = ds.size - - # read header - i, base_size = msb_size(db) - i, target_size = msb_size(db, i) - - # interpret opcodes - tbw = 0 # amount of target bytes written - while i < delta_buf_size: - c = ord(db[i]) - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = ord(db[i]) - i += 1 - if (c & 0x02): - cp_off |= (ord(db[i]) << 8) - i += 1 - if (c & 0x04): - cp_off |= (ord(db[i]) << 16) - i += 1 - if (c & 0x08): - cp_off |= (ord(db[i]) << 24) - i += 1 - if (c & 0x10): - cp_size = ord(db[i]) - i += 1 - if (c & 0x20): - cp_size |= (ord(db[i]) << 8) - i += 1 - if (c & 0x40): - cp_size |= (ord(db[i]) << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > base_size): - break - - dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) - tbw += cp_size - elif c: - # NOTE: in C, the data chunks should probably be concatenated here. - # In python, we do it as a post-process - dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) - i += c - tbw += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data - - dcl.compress() - - # merge the lists ! - if dsi > 0: - if not tdcl.connect_with_next_base(dcl): - break - # END handle merge - - # prepare next base - dcl = DeltaChunkList() - # END for each delta stream - - return tdcl - + """ + Read the condensed delta chunk information from dstream and merge its information + into a list of existing delta chunks + + :param dstreams: iterable of delta stream objects, the delta to be applied last + comes first, then all its ancestors in order + :return: DeltaChunkList, containing all operations to apply""" + tdcl = None # topmost dcl + + dcl = tdcl = TopdownDeltaChunkList() + for dsi, ds in enumerate(dstreams): + # print "Stream", dsi + db = ds.read() + delta_buf_size = ds.size + + # read header + i, base_size = msb_size(db) + i, target_size = msb_size(db, i) + + # interpret opcodes + tbw = 0 # amount of target bytes written + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > base_size): + break + + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) + tbw += cp_size + elif c: + # NOTE: in C, the data chunks should probably be concatenated here. + # In python, we do it as a post-process + dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) + i += c + tbw += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + dcl.compress() + + # merge the lists ! + if dsi > 0: + if not tdcl.connect_with_next_base(dcl): + break + # END handle merge + + # prepare next base + dcl = DeltaChunkList() + # END for each delta stream + + return tdcl + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): - """ - Apply data from a delta buffer using a source buffer to the target file - - :param src_buf: random access data from which the delta was created - :param src_buf_size: size of the source buffer in bytes - :param delta_buf_size: size fo the delta buffer in bytes - :param delta_buf: random access delta data - :param write: write method taking a chunk of bytes - - **Note:** transcribed to python from the similar routine in patch-delta.c""" - i = 0 - db = delta_buf - while i < delta_buf_size: - c = ord(db[i]) - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = ord(db[i]) - i += 1 - if (c & 0x02): - cp_off |= (ord(db[i]) << 8) - i += 1 - if (c & 0x04): - cp_off |= (ord(db[i]) << 16) - i += 1 - if (c & 0x08): - cp_off |= (ord(db[i]) << 24) - i += 1 - if (c & 0x10): - cp_size = ord(db[i]) - i += 1 - if (c & 0x20): - cp_size |= (ord(db[i]) << 8) - i += 1 - if (c & 0x40): - cp_size |= (ord(db[i]) << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > src_buf_size): - break - write(buffer(src_buf, cp_off, cp_size)) - elif c: - write(db[i:i+c]) - i += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data - - # yes, lets use the exact same error message that git uses :) - assert i == delta_buf_size, "delta replay has gone wild" - - + """ + Apply data from a delta buffer using a source buffer to the target file + + :param src_buf: random access data from which the delta was created + :param src_buf_size: size of the source buffer in bytes + :param delta_buf_size: size fo the delta buffer in bytes + :param delta_buf: random access delta data + :param write: write method taking a chunk of bytes + + **Note:** transcribed to python from the similar routine in patch-delta.c""" + i = 0 + db = delta_buf + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(buffer(src_buf, cp_off, cp_size)) + elif c: + write(db[i:i+c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + + # yes, lets use the exact same error message that git uses :) + assert i == delta_buf_size, "delta replay has gone wild" + + def is_equal_canonical_sha(canonical_length, match, sha1): - """ - :return: True if the given lhs and rhs 20 byte binary shas - The comparison will take the canonical_length of the match sha into account, - hence the comparison will only use the last 4 bytes for uneven canonical representations - :param match: less than 20 byte sha - :param sha1: 20 byte sha""" - binary_length = canonical_length/2 - if match[:binary_length] != sha1[:binary_length]: - return False - - if canonical_length - binary_length and \ - (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: - return False - # END handle uneven canonnical length - return True - + """ + :return: True if the given lhs and rhs 20 byte binary shas + The comparison will take the canonical_length of the match sha into account, + hence the comparison will only use the last 4 bytes for uneven canonical representations + :param match: less than 20 byte sha + :param sha1: 20 byte sha""" + binary_length = canonical_length/2 + if match[:binary_length] != sha1[:binary_length]: + return False + + if canonical_length - binary_length and \ + (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: + return False + # END handle uneven canonnical length + return True + #} END routines try: - # raise ImportError; # DEBUG - from _perf import connect_deltas + # raise ImportError; # DEBUG + from _perf import connect_deltas except ImportError: - pass + pass diff --git a/gitdb/pack.py b/gitdb/pack.py index c6d1cc313..48121f026 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -4,59 +4,59 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( - BadObject, - UnsupportedOperation, - ParseError - ) + BadObject, + UnsupportedOperation, + ParseError + ) from util import ( - zlib, - mman, - LazyMixin, - unpack_from, - bin_to_hex, - ) + zlib, + mman, + LazyMixin, + unpack_from, + bin_to_hex, + ) from fun import ( - create_pack_object_header, - pack_object_header_info, - is_equal_canonical_sha, - type_id_to_type_map, - write_object, - stream_copy, - chunk_size, - delta_types, - OFS_DELTA, - REF_DELTA, - msb_size - ) + create_pack_object_header, + pack_object_header_info, + is_equal_canonical_sha, + type_id_to_type_map, + write_object, + stream_copy, + chunk_size, + delta_types, + OFS_DELTA, + REF_DELTA, + msb_size + ) try: - from _perf import PackIndexFile_sha_to_index + from _perf import PackIndexFile_sha_to_index except ImportError: - pass + pass # END try c module -from base import ( # Amazing ! - OInfo, - OStream, - OPackInfo, - OPackStream, - ODeltaStream, - ODeltaPackInfo, - ODeltaPackStream, - ) +from base import ( # Amazing ! + OInfo, + OStream, + OPackInfo, + OPackStream, + ODeltaStream, + ODeltaPackInfo, + ODeltaPackStream, + ) from stream import ( - DecompressMemMapReader, - DeltaApplyReader, - Sha1Writer, - NullStream, - FlexibleSha1Writer - ) + DecompressMemMapReader, + DeltaApplyReader, + Sha1Writer, + NullStream, + FlexibleSha1Writer + ) from struct import ( - pack, - unpack, - ) + pack, + unpack, + ) from binascii import crc32 @@ -70,943 +70,943 @@ - + #{ Utilities def pack_object_at(cursor, offset, as_stream): - """ - :return: Tuple(abs_data_offset, PackInfo|PackStream) - an object of the correct type according to the type_id of the object. - If as_stream is True, the object will contain a stream, allowing the - data to be read decompressed. - :param data: random accessable data containing all required information - :parma offset: offset in to the data at which the object information is located - :param as_stream: if True, a stream object will be returned that can read - the data, otherwise you receive an info object only""" - data = cursor.use_region(offset).buffer() - type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) - total_rela_offset = None # set later, actual offset until data stream begins - delta_info = None - - # OFFSET DELTA - if type_id == OFS_DELTA: - i = data_rela_offset - c = ord(data[i]) - i += 1 - delta_offset = c & 0x7f - while c & 0x80: - c = ord(data[i]) - i += 1 - delta_offset += 1 - delta_offset = (delta_offset << 7) + (c & 0x7f) - # END character loop - delta_info = delta_offset - total_rela_offset = i - # REF DELTA - elif type_id == REF_DELTA: - total_rela_offset = data_rela_offset+20 - delta_info = data[data_rela_offset:total_rela_offset] - # BASE OBJECT - else: - # assume its a base object - total_rela_offset = data_rela_offset - # END handle type id - - abs_data_offset = offset + total_rela_offset - if as_stream: - stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) - if delta_info is None: - return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) - else: - return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) - else: - if delta_info is None: - return abs_data_offset, OPackInfo(offset, type_id, uncomp_size) - else: - return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) - # END handle info - # END handle stream + """ + :return: Tuple(abs_data_offset, PackInfo|PackStream) + an object of the correct type according to the type_id of the object. + If as_stream is True, the object will contain a stream, allowing the + data to be read decompressed. + :param data: random accessable data containing all required information + :parma offset: offset in to the data at which the object information is located + :param as_stream: if True, a stream object will be returned that can read + the data, otherwise you receive an info object only""" + data = cursor.use_region(offset).buffer() + type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) + total_rela_offset = None # set later, actual offset until data stream begins + delta_info = None + + # OFFSET DELTA + if type_id == OFS_DELTA: + i = data_rela_offset + c = ord(data[i]) + i += 1 + delta_offset = c & 0x7f + while c & 0x80: + c = ord(data[i]) + i += 1 + delta_offset += 1 + delta_offset = (delta_offset << 7) + (c & 0x7f) + # END character loop + delta_info = delta_offset + total_rela_offset = i + # REF DELTA + elif type_id == REF_DELTA: + total_rela_offset = data_rela_offset+20 + delta_info = data[data_rela_offset:total_rela_offset] + # BASE OBJECT + else: + # assume its a base object + total_rela_offset = data_rela_offset + # END handle type id + + abs_data_offset = offset + total_rela_offset + if as_stream: + stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) + if delta_info is None: + return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) + else: + return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) + else: + if delta_info is None: + return abs_data_offset, OPackInfo(offset, type_id, uncomp_size) + else: + return abs_data_offset, ODeltaPackInfo(offset, type_id, uncomp_size, delta_info) + # END handle info + # END handle stream def write_stream_to_pack(read, write, zstream, base_crc=None): - """Copy a stream as read from read function, zip it, and write the result. - Count the number of written bytes and return it - :param base_crc: if not None, the crc will be the base for all compressed data - we consecutively write and generate a crc32 from. If None, no crc will be generated - :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if base_crc - was false""" - br = 0 # bytes read - bw = 0 # bytes written - want_crc = base_crc is not None - crc = 0 - if want_crc: - crc = base_crc - #END initialize crc - - while True: - chunk = read(chunk_size) - br += len(chunk) - compressed = zstream.compress(chunk) - bw += len(compressed) - write(compressed) # cannot assume return value - - if want_crc: - crc = crc32(compressed, crc) - #END handle crc - - if len(chunk) != chunk_size: - break - #END copy loop - - compressed = zstream.flush() - bw += len(compressed) - write(compressed) - if want_crc: - crc = crc32(compressed, crc) - #END handle crc - - return (br, bw, crc) + """Copy a stream as read from read function, zip it, and write the result. + Count the number of written bytes and return it + :param base_crc: if not None, the crc will be the base for all compressed data + we consecutively write and generate a crc32 from. If None, no crc will be generated + :return: tuple(no bytes read, no bytes written, crc32) crc might be 0 if base_crc + was false""" + br = 0 # bytes read + bw = 0 # bytes written + want_crc = base_crc is not None + crc = 0 + if want_crc: + crc = base_crc + #END initialize crc + + while True: + chunk = read(chunk_size) + br += len(chunk) + compressed = zstream.compress(chunk) + bw += len(compressed) + write(compressed) # cannot assume return value + + if want_crc: + crc = crc32(compressed, crc) + #END handle crc + + if len(chunk) != chunk_size: + break + #END copy loop + + compressed = zstream.flush() + bw += len(compressed) + write(compressed) + if want_crc: + crc = crc32(compressed, crc) + #END handle crc + + return (br, bw, crc) #} END utilities class IndexWriter(object): - """Utility to cache index information, allowing to write all information later - in one go to the given stream - **Note:** currently only writes v2 indices""" - __slots__ = '_objs' - - def __init__(self): - self._objs = list() - - def append(self, binsha, crc, offset): - """Append one piece of object information""" - self._objs.append((binsha, crc, offset)) - - def write(self, pack_sha, write): - """Write the index file using the given write method - :param pack_sha: binary sha over the whole pack that we index - :return: sha1 binary sha over all index file contents""" - # sort for sha1 hash - self._objs.sort(key=lambda o: o[0]) - - sha_writer = FlexibleSha1Writer(write) - sha_write = sha_writer.write - sha_write(PackIndexFile.index_v2_signature) - sha_write(pack(">L", PackIndexFile.index_version_default)) - - # fanout - tmplist = list((0,)*256) # fanout or list with 64 bit offsets - for t in self._objs: - tmplist[ord(t[0][0])] += 1 - #END prepare fanout - for i in xrange(255): - v = tmplist[i] - sha_write(pack('>L', v)) - tmplist[i+1] += v - #END write each fanout entry - sha_write(pack('>L', tmplist[255])) - - # sha1 ordered - # save calls, that is push them into c - sha_write(''.join(t[0] for t in self._objs)) - - # crc32 - for t in self._objs: - sha_write(pack('>L', t[1]&0xffffffff)) - #END for each crc - - tmplist = list() - # offset 32 - for t in self._objs: - ofs = t[2] - if ofs > 0x7fffffff: - tmplist.append(ofs) - ofs = 0x80000000 + len(tmplist)-1 - #END hande 64 bit offsets - sha_write(pack('>L', ofs&0xffffffff)) - #END for each offset - - # offset 64 - for ofs in tmplist: - sha_write(pack(">Q", ofs)) - #END for each offset - - # trailer - assert(len(pack_sha) == 20) - sha_write(pack_sha) - sha = sha_writer.sha(as_hex=False) - write(sha) - return sha - - + """Utility to cache index information, allowing to write all information later + in one go to the given stream + **Note:** currently only writes v2 indices""" + __slots__ = '_objs' + + def __init__(self): + self._objs = list() + + def append(self, binsha, crc, offset): + """Append one piece of object information""" + self._objs.append((binsha, crc, offset)) + + def write(self, pack_sha, write): + """Write the index file using the given write method + :param pack_sha: binary sha over the whole pack that we index + :return: sha1 binary sha over all index file contents""" + # sort for sha1 hash + self._objs.sort(key=lambda o: o[0]) + + sha_writer = FlexibleSha1Writer(write) + sha_write = sha_writer.write + sha_write(PackIndexFile.index_v2_signature) + sha_write(pack(">L", PackIndexFile.index_version_default)) + + # fanout + tmplist = list((0,)*256) # fanout or list with 64 bit offsets + for t in self._objs: + tmplist[ord(t[0][0])] += 1 + #END prepare fanout + for i in xrange(255): + v = tmplist[i] + sha_write(pack('>L', v)) + tmplist[i+1] += v + #END write each fanout entry + sha_write(pack('>L', tmplist[255])) + + # sha1 ordered + # save calls, that is push them into c + sha_write(''.join(t[0] for t in self._objs)) + + # crc32 + for t in self._objs: + sha_write(pack('>L', t[1]&0xffffffff)) + #END for each crc + + tmplist = list() + # offset 32 + for t in self._objs: + ofs = t[2] + if ofs > 0x7fffffff: + tmplist.append(ofs) + ofs = 0x80000000 + len(tmplist)-1 + #END hande 64 bit offsets + sha_write(pack('>L', ofs&0xffffffff)) + #END for each offset + + # offset 64 + for ofs in tmplist: + sha_write(pack(">Q", ofs)) + #END for each offset + + # trailer + assert(len(pack_sha) == 20) + sha_write(pack_sha) + sha = sha_writer.sha(as_hex=False) + write(sha) + return sha + + class PackIndexFile(LazyMixin): - """A pack index provides offsets into the corresponding pack, allowing to find - locations for offsets faster.""" - - # Dont use slots as we dynamically bind functions for each version, need a dict for this - # The slots you see here are just to keep track of our instance variables - # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', - # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') + """A pack index provides offsets into the corresponding pack, allowing to find + locations for offsets faster.""" + + # Dont use slots as we dynamically bind functions for each version, need a dict for this + # The slots you see here are just to keep track of our instance variables + # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', + # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') - # used in v2 indices - _sha_list_offset = 8 + 1024 - index_v2_signature = '\377tOc' - index_version_default = 2 + # used in v2 indices + _sha_list_offset = 8 + 1024 + index_v2_signature = '\377tOc' + index_version_default = 2 - def __init__(self, indexpath): - super(PackIndexFile, self).__init__() - self._indexpath = indexpath - - def _set_cache_(self, attr): - if attr == "_packfile_checksum": - self._packfile_checksum = self._cursor.map()[-40:-20] - elif attr == "_packfile_checksum": - self._packfile_checksum = self._cursor.map()[-20:] - elif attr == "_cursor": - # Note: We don't lock the file when reading as we cannot be sure - # that we can actually write to the location - it could be a read-only - # alternate for instance - self._cursor = mman.make_cursor(self._indexpath).use_region() - # We will assume that the index will always fully fit into memory ! - if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): - raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (self._indexpath, self._cursor.file_size(), mman.window_size())) - #END assert window size - else: - # now its time to initialize everything - if we are here, someone wants - # to access the fanout table or related properties - - # CHECK VERSION - mmap = self._cursor.map() - self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 - if self._version == 2: - version_id = unpack_from(">L", mmap, 4)[0] - assert version_id == self._version, "Unsupported index version: %i" % version_id - # END assert version - - # SETUP FUNCTIONS - # setup our functions according to the actual version - for fname in ('entry', 'offset', 'sha', 'crc'): - setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) - # END for each function to initialize - - - # INITIALIZE DATA - # byte offset is 8 if version is 2, 0 otherwise - self._initialize() - # END handle attributes - + def __init__(self, indexpath): + super(PackIndexFile, self).__init__() + self._indexpath = indexpath + + def _set_cache_(self, attr): + if attr == "_packfile_checksum": + self._packfile_checksum = self._cursor.map()[-40:-20] + elif attr == "_packfile_checksum": + self._packfile_checksum = self._cursor.map()[-20:] + elif attr == "_cursor": + # Note: We don't lock the file when reading as we cannot be sure + # that we can actually write to the location - it could be a read-only + # alternate for instance + self._cursor = mman.make_cursor(self._indexpath).use_region() + # We will assume that the index will always fully fit into memory ! + if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): + raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (self._indexpath, self._cursor.file_size(), mman.window_size())) + #END assert window size + else: + # now its time to initialize everything - if we are here, someone wants + # to access the fanout table or related properties + + # CHECK VERSION + mmap = self._cursor.map() + self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 + if self._version == 2: + version_id = unpack_from(">L", mmap, 4)[0] + assert version_id == self._version, "Unsupported index version: %i" % version_id + # END assert version + + # SETUP FUNCTIONS + # setup our functions according to the actual version + for fname in ('entry', 'offset', 'sha', 'crc'): + setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) + # END for each function to initialize + + + # INITIALIZE DATA + # byte offset is 8 if version is 2, 0 otherwise + self._initialize() + # END handle attributes + - #{ Access V1 - - def _entry_v1(self, i): - """:return: tuple(offset, binsha, 0)""" - return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) - - def _offset_v1(self, i): - """see ``_offset_v2``""" - return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] - - def _sha_v1(self, i): - """see ``_sha_v2``""" - base = 1024 + (i*24)+4 - return self._cursor.map()[base:base+20] - - def _crc_v1(self, i): - """unsupported""" - return 0 - - #} END access V1 - - #{ Access V2 - def _entry_v2(self, i): - """:return: tuple(offset, binsha, crc)""" - return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) - - def _offset_v2(self, i): - """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only - be returned if the pack is larger than 4 GiB, or 2^32""" - offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] - - # if the high-bit is set, this indicates that we have to lookup the offset - # in the 64 bit region of the file. The current offset ( lower 31 bits ) - # are the index into it - if offset & 0x80000000: - offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] - # END handle 64 bit offset - - return offset - - def _sha_v2(self, i): - """:return: sha at the given index of this file index instance""" - base = self._sha_list_offset + i * 20 - return self._cursor.map()[base:base+20] - - def _crc_v2(self, i): - """:return: 4 bytes crc for the object at index i""" - return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] - - #} END access V2 - - #{ Initialization - - def _initialize(self): - """initialize base data""" - self._fanout_table = self._read_fanout((self._version == 2) * 8) - - if self._version == 2: - self._crc_list_offset = self._sha_list_offset + self.size() * 20 - self._pack_offset = self._crc_list_offset + self.size() * 4 - self._pack_64_offset = self._pack_offset + self.size() * 4 - # END setup base - - def _read_fanout(self, byte_offset): - """Generate a fanout table from our data""" - d = self._cursor.map() - out = list() - append = out.append - for i in range(256): - append(unpack_from('>L', d, byte_offset + i*4)[0]) - # END for each entry - return out - - #} END initialization - - #{ Properties - def version(self): - return self._version - - def size(self): - """:return: amount of objects referred to by this index""" - return self._fanout_table[255] - - def path(self): - """:return: path to the packindexfile""" - return self._indexpath - - def packfile_checksum(self): - """:return: 20 byte sha representing the sha1 hash of the pack file""" - return self._cursor.map()[-40:-20] - - def indexfile_checksum(self): - """:return: 20 byte sha representing the sha1 hash of this index file""" - return self._cursor.map()[-20:] - - def offsets(self): - """:return: sequence of all offsets in the order in which they were written - - **Note:** return value can be random accessed, but may be immmutable""" - if self._version == 2: - # read stream to array, convert to tuple - a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears - a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) - - # networkbyteorder to something array likes more - if sys.byteorder == 'little': - a.byteswap() - return a - else: - return tuple(self.offset(index) for index in xrange(self.size())) - # END handle version - - def sha_to_index(self, sha): - """ - :return: index usable with the ``offset`` or ``entry`` method, or None - if the sha was not found in this pack index - :param sha: 20 byte sha to lookup""" - first_byte = ord(sha[0]) - get_sha = self.sha - lo = 0 # lower index, the left bound of the bisection - if first_byte != 0: - lo = self._fanout_table[first_byte-1] - hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - - # bisect until we have the sha - while lo < hi: - mid = (lo + hi) / 2 - c = cmp(sha, get_sha(mid)) - if c < 0: - hi = mid - elif not c: - return mid - else: - lo = mid + 1 - # END handle midpoint - # END bisect - return None - - def partial_sha_to_index(self, partial_bin_sha, canonical_length): - """ - :return: index as in `sha_to_index` or None if the sha was not found in this - index file - :param partial_bin_sha: an at least two bytes of a partial binary sha - :param canonical_length: lenght of the original hexadecimal representation of the - given partial binary sha - :raise AmbiguousObjectName:""" - if len(partial_bin_sha) < 2: - raise ValueError("Require at least 2 bytes of partial sha") - - first_byte = ord(partial_bin_sha[0]) - get_sha = self.sha - lo = 0 # lower index, the left bound of the bisection - if first_byte != 0: - lo = self._fanout_table[first_byte-1] - hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - - # fill the partial to full 20 bytes - filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) - - # find lowest - while lo < hi: - mid = (lo + hi) / 2 - c = cmp(filled_sha, get_sha(mid)) - if c < 0: - hi = mid - elif not c: - # perfect match - lo = mid - break - else: - lo = mid + 1 - # END handle midpoint - # END bisect - - if lo < self.size(): - cur_sha = get_sha(lo) - if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): - next_sha = None - if lo+1 < self.size(): - next_sha = get_sha(lo+1) - if next_sha and next_sha == cur_sha: - raise AmbiguousObjectName(partial_bin_sha) - return lo - # END if we have a match - # END if we found something - return None - - if 'PackIndexFile_sha_to_index' in globals(): - # NOTE: Its just about 25% faster, the major bottleneck might be the attr - # accesses - def sha_to_index(self, sha): - return PackIndexFile_sha_to_index(self, sha) - # END redefine heavy-hitter with c version - - #} END properties - - + #{ Access V1 + + def _entry_v1(self, i): + """:return: tuple(offset, binsha, 0)""" + return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) + + def _offset_v1(self, i): + """see ``_offset_v2``""" + return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] + + def _sha_v1(self, i): + """see ``_sha_v2``""" + base = 1024 + (i*24)+4 + return self._cursor.map()[base:base+20] + + def _crc_v1(self, i): + """unsupported""" + return 0 + + #} END access V1 + + #{ Access V2 + def _entry_v2(self, i): + """:return: tuple(offset, binsha, crc)""" + return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) + + def _offset_v2(self, i): + """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only + be returned if the pack is larger than 4 GiB, or 2^32""" + offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] + + # if the high-bit is set, this indicates that we have to lookup the offset + # in the 64 bit region of the file. The current offset ( lower 31 bits ) + # are the index into it + if offset & 0x80000000: + offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] + # END handle 64 bit offset + + return offset + + def _sha_v2(self, i): + """:return: sha at the given index of this file index instance""" + base = self._sha_list_offset + i * 20 + return self._cursor.map()[base:base+20] + + def _crc_v2(self, i): + """:return: 4 bytes crc for the object at index i""" + return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] + + #} END access V2 + + #{ Initialization + + def _initialize(self): + """initialize base data""" + self._fanout_table = self._read_fanout((self._version == 2) * 8) + + if self._version == 2: + self._crc_list_offset = self._sha_list_offset + self.size() * 20 + self._pack_offset = self._crc_list_offset + self.size() * 4 + self._pack_64_offset = self._pack_offset + self.size() * 4 + # END setup base + + def _read_fanout(self, byte_offset): + """Generate a fanout table from our data""" + d = self._cursor.map() + out = list() + append = out.append + for i in range(256): + append(unpack_from('>L', d, byte_offset + i*4)[0]) + # END for each entry + return out + + #} END initialization + + #{ Properties + def version(self): + return self._version + + def size(self): + """:return: amount of objects referred to by this index""" + return self._fanout_table[255] + + def path(self): + """:return: path to the packindexfile""" + return self._indexpath + + def packfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of the pack file""" + return self._cursor.map()[-40:-20] + + def indexfile_checksum(self): + """:return: 20 byte sha representing the sha1 hash of this index file""" + return self._cursor.map()[-20:] + + def offsets(self): + """:return: sequence of all offsets in the order in which they were written + + **Note:** return value can be random accessed, but may be immmutable""" + if self._version == 2: + # read stream to array, convert to tuple + a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears + a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) + + # networkbyteorder to something array likes more + if sys.byteorder == 'little': + a.byteswap() + return a + else: + return tuple(self.offset(index) for index in xrange(self.size())) + # END handle version + + def sha_to_index(self, sha): + """ + :return: index usable with the ``offset`` or ``entry`` method, or None + if the sha was not found in this pack index + :param sha: 20 byte sha to lookup""" + first_byte = ord(sha[0]) + get_sha = self.sha + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte-1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + # bisect until we have the sha + while lo < hi: + mid = (lo + hi) / 2 + c = cmp(sha, get_sha(mid)) + if c < 0: + hi = mid + elif not c: + return mid + else: + lo = mid + 1 + # END handle midpoint + # END bisect + return None + + def partial_sha_to_index(self, partial_bin_sha, canonical_length): + """ + :return: index as in `sha_to_index` or None if the sha was not found in this + index file + :param partial_bin_sha: an at least two bytes of a partial binary sha + :param canonical_length: lenght of the original hexadecimal representation of the + given partial binary sha + :raise AmbiguousObjectName:""" + if len(partial_bin_sha) < 2: + raise ValueError("Require at least 2 bytes of partial sha") + + first_byte = ord(partial_bin_sha[0]) + get_sha = self.sha + lo = 0 # lower index, the left bound of the bisection + if first_byte != 0: + lo = self._fanout_table[first_byte-1] + hi = self._fanout_table[first_byte] # the upper, right bound of the bisection + + # fill the partial to full 20 bytes + filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) + + # find lowest + while lo < hi: + mid = (lo + hi) / 2 + c = cmp(filled_sha, get_sha(mid)) + if c < 0: + hi = mid + elif not c: + # perfect match + lo = mid + break + else: + lo = mid + 1 + # END handle midpoint + # END bisect + + if lo < self.size(): + cur_sha = get_sha(lo) + if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): + next_sha = None + if lo+1 < self.size(): + next_sha = get_sha(lo+1) + if next_sha and next_sha == cur_sha: + raise AmbiguousObjectName(partial_bin_sha) + return lo + # END if we have a match + # END if we found something + return None + + if 'PackIndexFile_sha_to_index' in globals(): + # NOTE: Its just about 25% faster, the major bottleneck might be the attr + # accesses + def sha_to_index(self, sha): + return PackIndexFile_sha_to_index(self, sha) + # END redefine heavy-hitter with c version + + #} END properties + + class PackFile(LazyMixin): - """A pack is a file written according to the Version 2 for git packs - - As we currently use memory maps, it could be assumed that the maximum size of - packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be - fine though. - - **Note:** at some point, this might be implemented using streams as well, or - streams are an alternate path in the case memory maps cannot be created - for some reason - one clearly doesn't want to read 10GB at once in that - case""" - - __slots__ = ('_packpath', '_cursor', '_size', '_version') - pack_signature = 0x5041434b # 'PACK' - pack_version_default = 2 - - # offset into our data at which the first object starts - first_object_offset = 3*4 # header bytes - footer_size = 20 # final sha - - def __init__(self, packpath): - self._packpath = packpath - - def _set_cache_(self, attr): - # we fill the whole cache, whichever attribute gets queried first - self._cursor = mman.make_cursor(self._packpath).use_region() - - # read the header information - type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) - - # TODO: figure out whether we should better keep the lock, or maybe - # add a .keep file instead ? - if type_id != self.pack_signature: - raise ParseError("Invalid pack signature: %i" % type_id) - - def _iter_objects(self, start_offset, as_stream=True): - """Handle the actual iteration of objects within this pack""" - c = self._cursor - content_size = c.file_size() - self.footer_size - cur_offset = start_offset or self.first_object_offset - - null = NullStream() - while cur_offset < content_size: - data_offset, ostream = pack_object_at(c, cur_offset, True) - # scrub the stream to the end - this decompresses the object, but yields - # the amount of compressed bytes we need to get to the next offset - - stream_copy(ostream.read, null.write, ostream.size, chunk_size) - cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() - - - # if a stream is requested, reset it beforehand - # Otherwise return the Stream object directly, its derived from the - # info object - if as_stream: - ostream.stream.seek(0) - yield ostream - # END until we have read everything - - #{ Pack Information - - def size(self): - """:return: The amount of objects stored in this pack""" - return self._size - - def version(self): - """:return: the version of this pack""" - return self._version - - def data(self): - """ - :return: read-only data of this pack. It provides random access and usually - is a memory map. - :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" - # can use map as we are starting at offset 0. Otherwise we would have to use buffer() - return self._cursor.use_region().map() - - def checksum(self): - """:return: 20 byte sha1 hash on all object sha's contained in this file""" - return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] - - def path(self): - """:return: path to the packfile""" - return self._packpath - #} END pack information - - #{ Pack Specific - - def collect_streams(self, offset): - """ - :return: list of pack streams which are required to build the object - at the given offset. The first entry of the list is the object at offset, - the last one is either a full object, or a REF_Delta stream. The latter - type needs its reference object to be locked up in an ODB to form a valid - delta chain. - If the object at offset is no delta, the size of the list is 1. - :param offset: specifies the first byte of the object within this pack""" - out = list() - c = self._cursor - while True: - ostream = pack_object_at(c, offset, True)[1] - out.append(ostream) - if ostream.type_id == OFS_DELTA: - offset = ostream.pack_offset - ostream.delta_info - else: - # the only thing we can lookup are OFFSET deltas. Everything - # else is either an object, or a ref delta, in the latter - # case someone else has to find it - break - # END handle type - # END while chaining streams - return out + """A pack is a file written according to the Version 2 for git packs + + As we currently use memory maps, it could be assumed that the maximum size of + packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be + fine though. + + **Note:** at some point, this might be implemented using streams as well, or + streams are an alternate path in the case memory maps cannot be created + for some reason - one clearly doesn't want to read 10GB at once in that + case""" + + __slots__ = ('_packpath', '_cursor', '_size', '_version') + pack_signature = 0x5041434b # 'PACK' + pack_version_default = 2 + + # offset into our data at which the first object starts + first_object_offset = 3*4 # header bytes + footer_size = 20 # final sha + + def __init__(self, packpath): + self._packpath = packpath + + def _set_cache_(self, attr): + # we fill the whole cache, whichever attribute gets queried first + self._cursor = mman.make_cursor(self._packpath).use_region() + + # read the header information + type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) + + # TODO: figure out whether we should better keep the lock, or maybe + # add a .keep file instead ? + if type_id != self.pack_signature: + raise ParseError("Invalid pack signature: %i" % type_id) + + def _iter_objects(self, start_offset, as_stream=True): + """Handle the actual iteration of objects within this pack""" + c = self._cursor + content_size = c.file_size() - self.footer_size + cur_offset = start_offset or self.first_object_offset + + null = NullStream() + while cur_offset < content_size: + data_offset, ostream = pack_object_at(c, cur_offset, True) + # scrub the stream to the end - this decompresses the object, but yields + # the amount of compressed bytes we need to get to the next offset + + stream_copy(ostream.read, null.write, ostream.size, chunk_size) + cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() + + + # if a stream is requested, reset it beforehand + # Otherwise return the Stream object directly, its derived from the + # info object + if as_stream: + ostream.stream.seek(0) + yield ostream + # END until we have read everything + + #{ Pack Information + + def size(self): + """:return: The amount of objects stored in this pack""" + return self._size + + def version(self): + """:return: the version of this pack""" + return self._version + + def data(self): + """ + :return: read-only data of this pack. It provides random access and usually + is a memory map. + :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" + # can use map as we are starting at offset 0. Otherwise we would have to use buffer() + return self._cursor.use_region().map() + + def checksum(self): + """:return: 20 byte sha1 hash on all object sha's contained in this file""" + return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] + + def path(self): + """:return: path to the packfile""" + return self._packpath + #} END pack information + + #{ Pack Specific + + def collect_streams(self, offset): + """ + :return: list of pack streams which are required to build the object + at the given offset. The first entry of the list is the object at offset, + the last one is either a full object, or a REF_Delta stream. The latter + type needs its reference object to be locked up in an ODB to form a valid + delta chain. + If the object at offset is no delta, the size of the list is 1. + :param offset: specifies the first byte of the object within this pack""" + out = list() + c = self._cursor + while True: + ostream = pack_object_at(c, offset, True)[1] + out.append(ostream) + if ostream.type_id == OFS_DELTA: + offset = ostream.pack_offset - ostream.delta_info + else: + # the only thing we can lookup are OFFSET deltas. Everything + # else is either an object, or a ref delta, in the latter + # case someone else has to find it + break + # END handle type + # END while chaining streams + return out - #} END pack specific - - #{ Read-Database like Interface - - def info(self, offset): - """Retrieve information about the object at the given file-absolute offset - - :param offset: byte offset - :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] - - def stream(self, offset): - """Retrieve an object at the given file-relative offset as stream along with its information - - :param offset: byte offset - :return: OPackStream instance, the actual type differs depending on the type_id attribute""" - return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] - - def stream_iter(self, start_offset=0): - """ - :return: iterator yielding OPackStream compatible instances, allowing - to access the data in the pack directly. - :param start_offset: offset to the first object to iterate. If 0, iteration - starts at the very first object in the pack. - - **Note:** Iterating a pack directly is costly as the datastream has to be decompressed - to determine the bounds between the objects""" - return self._iter_objects(start_offset, as_stream=True) - - #} END Read-Database like Interface - - + #} END pack specific + + #{ Read-Database like Interface + + def info(self, offset): + """Retrieve information about the object at the given file-absolute offset + + :param offset: byte offset + :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" + return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] + + def stream(self, offset): + """Retrieve an object at the given file-relative offset as stream along with its information + + :param offset: byte offset + :return: OPackStream instance, the actual type differs depending on the type_id attribute""" + return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] + + def stream_iter(self, start_offset=0): + """ + :return: iterator yielding OPackStream compatible instances, allowing + to access the data in the pack directly. + :param start_offset: offset to the first object to iterate. If 0, iteration + starts at the very first object in the pack. + + **Note:** Iterating a pack directly is costly as the datastream has to be decompressed + to determine the bounds between the objects""" + return self._iter_objects(start_offset, as_stream=True) + + #} END Read-Database like Interface + + class PackEntity(LazyMixin): - """Combines the PackIndexFile and the PackFile into one, allowing the - actual objects to be resolved and iterated""" - - __slots__ = ( '_index', # our index file - '_pack', # our pack file - '_offset_map' # on demand dict mapping one offset to the next consecutive one - ) - - IndexFileCls = PackIndexFile - PackFileCls = PackFile - - def __init__(self, pack_or_index_path): - """Initialize ourselves with the path to the respective pack or index file""" - basename, ext = os.path.splitext(pack_or_index_path) - self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance - self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance - - def _set_cache_(self, attr): - # currently this can only be _offset_map - # TODO: make this a simple sorted offset array which can be bisected - # to find the respective entry, from which we can take a +1 easily - # This might be slower, but should also be much lighter in memory ! - offsets_sorted = sorted(self._index.offsets()) - last_offset = len(self._pack.data()) - self._pack.footer_size - assert offsets_sorted, "Cannot handle empty indices" - - offset_map = None - if len(offsets_sorted) == 1: - offset_map = { offsets_sorted[0] : last_offset } - else: - iter_offsets = iter(offsets_sorted) - iter_offsets_plus_one = iter(offsets_sorted) - iter_offsets_plus_one.next() - consecutive = izip(iter_offsets, iter_offsets_plus_one) - - offset_map = dict(consecutive) - - # the last offset is not yet set - offset_map[offsets_sorted[-1]] = last_offset - # END handle offset amount - self._offset_map = offset_map - - def _sha_to_index(self, sha): - """:return: index for the given sha, or raise""" - index = self._index.sha_to_index(sha) - if index is None: - raise BadObject(sha) - return index - - def _iter_objects(self, as_stream): - """Iterate over all objects in our index and yield their OInfo or OStream instences""" - _sha = self._index.sha - _object = self._object - for index in xrange(self._index.size()): - yield _object(_sha(index), as_stream, index) - # END for each index - - def _object(self, sha, as_stream, index=-1): - """:return: OInfo or OStream object providing information about the given sha - :param index: if not -1, its assumed to be the sha's index in the IndexFile""" - # its a little bit redundant here, but it needs to be efficient - if index < 0: - index = self._sha_to_index(sha) - if sha is None: - sha = self._index.sha(index) - # END assure sha is present ( in output ) - offset = self._index.offset(index) - type_id, uncomp_size, data_rela_offset = pack_object_header_info(self._pack._cursor.use_region(offset).buffer()) - if as_stream: - if type_id not in delta_types: - packstream = self._pack.stream(offset) - return OStream(sha, packstream.type, packstream.size, packstream.stream) - # END handle non-deltas - - # produce a delta stream containing all info - # To prevent it from applying the deltas when querying the size, - # we extract it from the delta stream ourselves - streams = self.collect_streams_at_offset(offset) - dstream = DeltaApplyReader.new(streams) - - return ODeltaStream(sha, dstream.type, None, dstream) - else: - if type_id not in delta_types: - return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) - # END handle non-deltas - - # deltas are a little tougher - unpack the first bytes to obtain - # the actual target size, as opposed to the size of the delta data - streams = self.collect_streams_at_offset(offset) - buf = streams[0].read(512) - offset, src_size = msb_size(buf) - offset, target_size = msb_size(buf, offset) - - # collect the streams to obtain the actual object type - if streams[-1].type_id in delta_types: - raise BadObject(sha, "Could not resolve delta object") - return OInfo(sha, streams[-1].type, target_size) - # END handle stream - - #{ Read-Database like Interface - - def info(self, sha): - """Retrieve information about the object identified by the given sha - - :param sha: 20 byte sha1 - :raise BadObject: - :return: OInfo instance, with 20 byte sha""" - return self._object(sha, False) - - def stream(self, sha): - """Retrieve an object stream along with its information as identified by the given sha - - :param sha: 20 byte sha1 - :raise BadObject: - :return: OStream instance, with 20 byte sha""" - return self._object(sha, True) + """Combines the PackIndexFile and the PackFile into one, allowing the + actual objects to be resolved and iterated""" + + __slots__ = ( '_index', # our index file + '_pack', # our pack file + '_offset_map' # on demand dict mapping one offset to the next consecutive one + ) + + IndexFileCls = PackIndexFile + PackFileCls = PackFile + + def __init__(self, pack_or_index_path): + """Initialize ourselves with the path to the respective pack or index file""" + basename, ext = os.path.splitext(pack_or_index_path) + self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance + self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + + def _set_cache_(self, attr): + # currently this can only be _offset_map + # TODO: make this a simple sorted offset array which can be bisected + # to find the respective entry, from which we can take a +1 easily + # This might be slower, but should also be much lighter in memory ! + offsets_sorted = sorted(self._index.offsets()) + last_offset = len(self._pack.data()) - self._pack.footer_size + assert offsets_sorted, "Cannot handle empty indices" + + offset_map = None + if len(offsets_sorted) == 1: + offset_map = { offsets_sorted[0] : last_offset } + else: + iter_offsets = iter(offsets_sorted) + iter_offsets_plus_one = iter(offsets_sorted) + iter_offsets_plus_one.next() + consecutive = izip(iter_offsets, iter_offsets_plus_one) + + offset_map = dict(consecutive) + + # the last offset is not yet set + offset_map[offsets_sorted[-1]] = last_offset + # END handle offset amount + self._offset_map = offset_map + + def _sha_to_index(self, sha): + """:return: index for the given sha, or raise""" + index = self._index.sha_to_index(sha) + if index is None: + raise BadObject(sha) + return index + + def _iter_objects(self, as_stream): + """Iterate over all objects in our index and yield their OInfo or OStream instences""" + _sha = self._index.sha + _object = self._object + for index in xrange(self._index.size()): + yield _object(_sha(index), as_stream, index) + # END for each index + + def _object(self, sha, as_stream, index=-1): + """:return: OInfo or OStream object providing information about the given sha + :param index: if not -1, its assumed to be the sha's index in the IndexFile""" + # its a little bit redundant here, but it needs to be efficient + if index < 0: + index = self._sha_to_index(sha) + if sha is None: + sha = self._index.sha(index) + # END assure sha is present ( in output ) + offset = self._index.offset(index) + type_id, uncomp_size, data_rela_offset = pack_object_header_info(self._pack._cursor.use_region(offset).buffer()) + if as_stream: + if type_id not in delta_types: + packstream = self._pack.stream(offset) + return OStream(sha, packstream.type, packstream.size, packstream.stream) + # END handle non-deltas + + # produce a delta stream containing all info + # To prevent it from applying the deltas when querying the size, + # we extract it from the delta stream ourselves + streams = self.collect_streams_at_offset(offset) + dstream = DeltaApplyReader.new(streams) + + return ODeltaStream(sha, dstream.type, None, dstream) + else: + if type_id not in delta_types: + return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) + # END handle non-deltas + + # deltas are a little tougher - unpack the first bytes to obtain + # the actual target size, as opposed to the size of the delta data + streams = self.collect_streams_at_offset(offset) + buf = streams[0].read(512) + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + + # collect the streams to obtain the actual object type + if streams[-1].type_id in delta_types: + raise BadObject(sha, "Could not resolve delta object") + return OInfo(sha, streams[-1].type, target_size) + # END handle stream + + #{ Read-Database like Interface + + def info(self, sha): + """Retrieve information about the object identified by the given sha + + :param sha: 20 byte sha1 + :raise BadObject: + :return: OInfo instance, with 20 byte sha""" + return self._object(sha, False) + + def stream(self, sha): + """Retrieve an object stream along with its information as identified by the given sha + + :param sha: 20 byte sha1 + :raise BadObject: + :return: OStream instance, with 20 byte sha""" + return self._object(sha, True) - def info_at_index(self, index): - """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" - return self._object(None, False, index) - - def stream_at_index(self, index): - """As ``stream``, but uses a PackIndexFile compatible index to refer to the - object""" - return self._object(None, True, index) - - #} END Read-Database like Interface - - #{ Interface + def info_at_index(self, index): + """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" + return self._object(None, False, index) + + def stream_at_index(self, index): + """As ``stream``, but uses a PackIndexFile compatible index to refer to the + object""" + return self._object(None, True, index) + + #} END Read-Database like Interface + + #{ Interface - def pack(self): - """:return: the underlying pack file instance""" - return self._pack - - def index(self): - """:return: the underlying pack index file instance""" - return self._index - - def is_valid_stream(self, sha, use_crc=False): - """ - Verify that the stream at the given sha is valid. - - :param use_crc: if True, the index' crc is run over the compressed stream of - the object, which is much faster than checking the sha1. It is also - more prone to unnoticed corruption or manipulation. - :param sha: 20 byte sha1 of the object whose stream to verify - whether the compressed stream of the object is valid. If it is - a delta, this only verifies that the delta's data is valid, not the - data of the actual undeltified object, as it depends on more than - just this stream. - If False, the object will be decompressed and the sha generated. It must - match the given sha - - :return: True if the stream is valid - :raise UnsupportedOperation: If the index is version 1 only - :raise BadObject: sha was not found""" - if use_crc: - if self._index.version() < 2: - raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") - # END handle index version - - index = self._sha_to_index(sha) - offset = self._index.offset(index) - next_offset = self._offset_map[offset] - crc_value = self._index.crc(index) - - # create the current crc value, on the compressed object data - # Read it in chunks, without copying the data - crc_update = zlib.crc32 - pack_data = self._pack.data() - cur_pos = offset - this_crc_value = 0 - while cur_pos < next_offset: - rbound = min(cur_pos + chunk_size, next_offset) - size = rbound - cur_pos - this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) - cur_pos += size - # END window size loop - - # crc returns signed 32 bit numbers, the AND op forces it into unsigned - # mode ... wow, sneaky, from dulwich. - return (this_crc_value & 0xffffffff) == crc_value - else: - shawriter = Sha1Writer() - stream = self._object(sha, as_stream=True) - # write a loose object, which is the basis for the sha - write_object(stream.type, stream.size, stream.read, shawriter.write) - - assert shawriter.sha(as_hex=False) == sha - return shawriter.sha(as_hex=False) == sha - # END handle crc/sha verification - return True + def pack(self): + """:return: the underlying pack file instance""" + return self._pack + + def index(self): + """:return: the underlying pack index file instance""" + return self._index + + def is_valid_stream(self, sha, use_crc=False): + """ + Verify that the stream at the given sha is valid. + + :param use_crc: if True, the index' crc is run over the compressed stream of + the object, which is much faster than checking the sha1. It is also + more prone to unnoticed corruption or manipulation. + :param sha: 20 byte sha1 of the object whose stream to verify + whether the compressed stream of the object is valid. If it is + a delta, this only verifies that the delta's data is valid, not the + data of the actual undeltified object, as it depends on more than + just this stream. + If False, the object will be decompressed and the sha generated. It must + match the given sha + + :return: True if the stream is valid + :raise UnsupportedOperation: If the index is version 1 only + :raise BadObject: sha was not found""" + if use_crc: + if self._index.version() < 2: + raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") + # END handle index version + + index = self._sha_to_index(sha) + offset = self._index.offset(index) + next_offset = self._offset_map[offset] + crc_value = self._index.crc(index) + + # create the current crc value, on the compressed object data + # Read it in chunks, without copying the data + crc_update = zlib.crc32 + pack_data = self._pack.data() + cur_pos = offset + this_crc_value = 0 + while cur_pos < next_offset: + rbound = min(cur_pos + chunk_size, next_offset) + size = rbound - cur_pos + this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) + cur_pos += size + # END window size loop + + # crc returns signed 32 bit numbers, the AND op forces it into unsigned + # mode ... wow, sneaky, from dulwich. + return (this_crc_value & 0xffffffff) == crc_value + else: + shawriter = Sha1Writer() + stream = self._object(sha, as_stream=True) + # write a loose object, which is the basis for the sha + write_object(stream.type, stream.size, stream.read, shawriter.write) + + assert shawriter.sha(as_hex=False) == sha + return shawriter.sha(as_hex=False) == sha + # END handle crc/sha verification + return True - def info_iter(self): - """ - :return: Iterator over all objects in this pack. The iterator yields - OInfo instances""" - return self._iter_objects(as_stream=False) - - def stream_iter(self): - """ - :return: iterator over all objects in this pack. The iterator yields - OStream instances""" - return self._iter_objects(as_stream=True) - - def collect_streams_at_offset(self, offset): - """ - As the version in the PackFile, but can resolve REF deltas within this pack - For more info, see ``collect_streams`` - - :param offset: offset into the pack file at which the object can be found""" - streams = self._pack.collect_streams(offset) - - # try to resolve the last one if needed. It is assumed to be either - # a REF delta, or a base object, as OFFSET deltas are resolved by the pack - if streams[-1].type_id == REF_DELTA: - stream = streams[-1] - while stream.type_id in delta_types: - if stream.type_id == REF_DELTA: - sindex = self._index.sha_to_index(stream.delta_info) - if sindex is None: - break - stream = self._pack.stream(self._index.offset(sindex)) - streams.append(stream) - else: - # must be another OFS DELTA - this could happen if a REF - # delta we resolve previously points to an OFS delta. Who - # would do that ;) ? We can handle it though - stream = self._pack.stream(stream.delta_info) - streams.append(stream) - # END handle ref delta - # END resolve ref streams - # END resolve streams - - return streams - - def collect_streams(self, sha): - """ - As ``PackFile.collect_streams``, but takes a sha instead of an offset. - Additionally, ref_delta streams will be resolved within this pack. - If this is not possible, the stream will be left alone, hence it is adivsed - to check for unresolved ref-deltas and resolve them before attempting to - construct a delta stream. - - :param sha: 20 byte sha1 specifying the object whose related streams you want to collect - :return: list of streams, first being the actual object delta, the last being - a possibly unresolved base object. - :raise BadObject:""" - return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) - - - @classmethod - def write_pack(cls, object_iter, pack_write, index_write=None, - object_count = None, zlib_compression = zlib.Z_BEST_SPEED): - """ - Create a new pack by putting all objects obtained by the object_iterator - into a pack which is written using the pack_write method. - The respective index is produced as well if index_write is not Non. - - :param object_iter: iterator yielding odb output objects - :param pack_write: function to receive strings to write into the pack stream - :param indx_write: if not None, the function writes the index file corresponding - to the pack. - :param object_count: if you can provide the amount of objects in your iteration, - this would be the place to put it. Otherwise we have to pre-iterate and store - all items into a list to get the number, which uses more memory than necessary. - :param zlib_compression: the zlib compression level to use - :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack - and over all contents of the index. If index_write was None, index_binsha will be None - - **Note:** The destination of the write functions is up to the user. It could - be a socket, or a file for instance - - **Note:** writes only undeltified objects""" - objs = object_iter - if not object_count: - if not isinstance(object_iter, (tuple, list)): - objs = list(object_iter) - #END handle list type - object_count = len(objs) - #END handle object - - pack_writer = FlexibleSha1Writer(pack_write) - pwrite = pack_writer.write - ofs = 0 # current offset into the pack file - index = None - wants_index = index_write is not None - - # write header - pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) - ofs += 12 - - if wants_index: - index = IndexWriter() - #END handle index header - - actual_count = 0 - for obj in objs: - actual_count += 1 - crc = 0 - - # object header - hdr = create_pack_object_header(obj.type_id, obj.size) - if index_write: - crc = crc32(hdr) - else: - crc = None - #END handle crc - pwrite(hdr) - - # data stream - zstream = zlib.compressobj(zlib_compression) - ostream = obj.stream - br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc = crc) - assert(br == obj.size) - if wants_index: - index.append(obj.binsha, crc, ofs) - #END handle index - - ofs += len(hdr) + bw - if actual_count == object_count: - break - #END abort once we are done - #END for each object - - if actual_count != object_count: - raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) - #END count assertion - - # write footer - pack_sha = pack_writer.sha(as_hex = False) - assert len(pack_sha) == 20 - pack_write(pack_sha) - ofs += len(pack_sha) # just for completeness ;) - - index_sha = None - if wants_index: - index_sha = index.write(pack_sha, index_write) - #END handle index - - return pack_sha, index_sha - - @classmethod - def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): - """Create a new on-disk entity comprised of a properly named pack file and a properly named - and corresponding index file. The pack contains all OStream objects contained in object iter. - :param base_dir: directory which is to contain the files - :return: PackEntity instance initialized with the new pack - - **Note:** for more information on the other parameters see the write_pack method""" - pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) - index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) - pack_write = lambda d: os.write(pack_fd, d) - index_write = lambda d: os.write(index_fd, d) - - pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) - os.close(pack_fd) - os.close(index_fd) - - fmt = "pack-%s.%s" - new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) - new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) - os.rename(pack_path, new_pack_path) - os.rename(index_path, new_index_path) - - return cls(new_pack_path) - - - #} END interface + def info_iter(self): + """ + :return: Iterator over all objects in this pack. The iterator yields + OInfo instances""" + return self._iter_objects(as_stream=False) + + def stream_iter(self): + """ + :return: iterator over all objects in this pack. The iterator yields + OStream instances""" + return self._iter_objects(as_stream=True) + + def collect_streams_at_offset(self, offset): + """ + As the version in the PackFile, but can resolve REF deltas within this pack + For more info, see ``collect_streams`` + + :param offset: offset into the pack file at which the object can be found""" + streams = self._pack.collect_streams(offset) + + # try to resolve the last one if needed. It is assumed to be either + # a REF delta, or a base object, as OFFSET deltas are resolved by the pack + if streams[-1].type_id == REF_DELTA: + stream = streams[-1] + while stream.type_id in delta_types: + if stream.type_id == REF_DELTA: + sindex = self._index.sha_to_index(stream.delta_info) + if sindex is None: + break + stream = self._pack.stream(self._index.offset(sindex)) + streams.append(stream) + else: + # must be another OFS DELTA - this could happen if a REF + # delta we resolve previously points to an OFS delta. Who + # would do that ;) ? We can handle it though + stream = self._pack.stream(stream.delta_info) + streams.append(stream) + # END handle ref delta + # END resolve ref streams + # END resolve streams + + return streams + + def collect_streams(self, sha): + """ + As ``PackFile.collect_streams``, but takes a sha instead of an offset. + Additionally, ref_delta streams will be resolved within this pack. + If this is not possible, the stream will be left alone, hence it is adivsed + to check for unresolved ref-deltas and resolve them before attempting to + construct a delta stream. + + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect + :return: list of streams, first being the actual object delta, the last being + a possibly unresolved base object. + :raise BadObject:""" + return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) + + + @classmethod + def write_pack(cls, object_iter, pack_write, index_write=None, + object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + """ + Create a new pack by putting all objects obtained by the object_iterator + into a pack which is written using the pack_write method. + The respective index is produced as well if index_write is not Non. + + :param object_iter: iterator yielding odb output objects + :param pack_write: function to receive strings to write into the pack stream + :param indx_write: if not None, the function writes the index file corresponding + to the pack. + :param object_count: if you can provide the amount of objects in your iteration, + this would be the place to put it. Otherwise we have to pre-iterate and store + all items into a list to get the number, which uses more memory than necessary. + :param zlib_compression: the zlib compression level to use + :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack + and over all contents of the index. If index_write was None, index_binsha will be None + + **Note:** The destination of the write functions is up to the user. It could + be a socket, or a file for instance + + **Note:** writes only undeltified objects""" + objs = object_iter + if not object_count: + if not isinstance(object_iter, (tuple, list)): + objs = list(object_iter) + #END handle list type + object_count = len(objs) + #END handle object + + pack_writer = FlexibleSha1Writer(pack_write) + pwrite = pack_writer.write + ofs = 0 # current offset into the pack file + index = None + wants_index = index_write is not None + + # write header + pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) + ofs += 12 + + if wants_index: + index = IndexWriter() + #END handle index header + + actual_count = 0 + for obj in objs: + actual_count += 1 + crc = 0 + + # object header + hdr = create_pack_object_header(obj.type_id, obj.size) + if index_write: + crc = crc32(hdr) + else: + crc = None + #END handle crc + pwrite(hdr) + + # data stream + zstream = zlib.compressobj(zlib_compression) + ostream = obj.stream + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc = crc) + assert(br == obj.size) + if wants_index: + index.append(obj.binsha, crc, ofs) + #END handle index + + ofs += len(hdr) + bw + if actual_count == object_count: + break + #END abort once we are done + #END for each object + + if actual_count != object_count: + raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) + #END count assertion + + # write footer + pack_sha = pack_writer.sha(as_hex = False) + assert len(pack_sha) == 20 + pack_write(pack_sha) + ofs += len(pack_sha) # just for completeness ;) + + index_sha = None + if wants_index: + index_sha = index.write(pack_sha, index_write) + #END handle index + + return pack_sha, index_sha + + @classmethod + def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + """Create a new on-disk entity comprised of a properly named pack file and a properly named + and corresponding index file. The pack contains all OStream objects contained in object iter. + :param base_dir: directory which is to contain the files + :return: PackEntity instance initialized with the new pack + + **Note:** for more information on the other parameters see the write_pack method""" + pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) + index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) + pack_write = lambda d: os.write(pack_fd, d) + index_write = lambda d: os.write(index_fd, d) + + pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) + os.close(pack_fd) + os.close(index_fd) + + fmt = "pack-%s.%s" + new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) + new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) + os.rename(pack_path, new_pack_path) + os.rename(index_path, new_index_path) + + return cls(new_pack_path) + + + #} END interface diff --git a/gitdb/stream.py b/gitdb/stream.py index 632213c27..6441b1e1a 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -9,684 +9,684 @@ import os from fun import ( - msb_size, - stream_copy, - apply_delta_data, - connect_deltas, - DeltaChunkList, - delta_types - ) + msb_size, + stream_copy, + apply_delta_data, + connect_deltas, + DeltaChunkList, + delta_types + ) from util import ( - allocate_memory, - LazyMixin, - make_sha, - write, - close, - zlib - ) + allocate_memory, + LazyMixin, + make_sha, + write, + close, + zlib + ) has_perf_mod = False try: - from _perf import apply_delta as c_apply_delta - has_perf_mod = True + from _perf import apply_delta as c_apply_delta + has_perf_mod = True except ImportError: - pass + pass -__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', - 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', - 'FDStream', 'NullStream') +__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', + 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', + 'FDStream', 'NullStream') #{ RO Streams class DecompressMemMapReader(LazyMixin): - """Reads data in chunks from a memory map and decompresses it. The client sees - only the uncompressed data, respective file-like read calls are handling on-demand - buffered decompression accordingly - - A constraint on the total size of bytes is activated, simulating - a logical file within a possibly larger physical memory area - - To read efficiently, you clearly don't want to read individual bytes, instead, - read a few kilobytes at least. - - **Note:** The chunk-size should be carefully selected as it will involve quite a bit - of string copying due to the way the zlib is implemented. Its very wasteful, - hence we try to find a good tradeoff between allocation time and number of - times we actually allocate. An own zlib implementation would be good here - to better support streamed reading - it would only need to keep the mmap - and decompress it into chunks, thats all ... """ - __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', - '_cbr', '_phi') - - max_read_size = 512*1024 # currently unused - - def __init__(self, m, close_on_deletion, size=None): - """Initialize with mmap for stream reading - :param m: must be content data - use new if you have object data and no size""" - self._m = m - self._zip = zlib.decompressobj() - self._buf = None # buffer of decompressed bytes - self._buflen = 0 # length of bytes in buffer - if size is not None: - self._s = size # size of uncompressed data to read in total - self._br = 0 # num uncompressed bytes read - self._cws = 0 # start byte of compression window - self._cwe = 0 # end byte of compression window - self._cbr = 0 # number of compressed bytes read - self._phi = False # is True if we parsed the header info - self._close = close_on_deletion # close the memmap on deletion ? - - def _set_cache_(self, attr): - assert attr == '_s' - # only happens for size, which is a marker to indicate we still - # have to parse the header from the stream - self._parse_header_info() - - def __del__(self): - if self._close: - self._m.close() - # END handle resource freeing - - def _parse_header_info(self): - """If this stream contains object data, parse the header info and skip the - stream to a point where each read will yield object content - - :return: parsed type_string, size""" - # read header - maxb = 512 # should really be enough, cgit uses 8192 I believe - self._s = maxb - hdr = self.read(maxb) - hdrend = hdr.find("\0") - type, size = hdr[:hdrend].split(" ") - size = int(size) - self._s = size - - # adjust internal state to match actual header length that we ignore - # The buffer will be depleted first on future reads - self._br = 0 - hdrend += 1 # count terminating \0 - self._buf = StringIO(hdr[hdrend:]) - self._buflen = len(hdr) - hdrend - - self._phi = True - - return type, size - - #{ Interface - - @classmethod - def new(self, m, close_on_deletion=False): - """Create a new DecompressMemMapReader instance for acting as a read-only stream - This method parses the object header from m and returns the parsed - type and size, as well as the created stream instance. - - :param m: memory map on which to oparate. It must be object data ( header + contents ) - :param close_on_deletion: if True, the memory map will be closed once we are - being deleted""" - inst = DecompressMemMapReader(m, close_on_deletion, 0) - type, size = inst._parse_header_info() - return type, size, inst - - def data(self): - """:return: random access compatible data we are working on""" - return self._m - - def compressed_bytes_read(self): - """ - :return: number of compressed bytes read. This includes the bytes it - took to decompress the header ( if there was one )""" - # ABSTRACT: When decompressing a byte stream, it can be that the first - # x bytes which were requested match the first x bytes in the loosely - # compressed datastream. This is the worst-case assumption that the reader - # does, it assumes that it will get at least X bytes from X compressed bytes - # in call cases. - # The caveat is that the object, according to our known uncompressed size, - # is already complete, but there are still some bytes left in the compressed - # stream that contribute to the amount of compressed bytes. - # How can we know that we are truly done, and have read all bytes we need - # to read ? - # Without help, we cannot know, as we need to obtain the status of the - # decompression. If it is not finished, we need to decompress more data - # until it is finished, to yield the actual number of compressed bytes - # belonging to the decompressed object - # We are using a custom zlib module for this, if its not present, - # we try to put in additional bytes up for decompression if feasible - # and check for the unused_data. - - # Only scrub the stream forward if we are officially done with the - # bytes we were to have. - if self._br == self._s and not self._zip.unused_data: - # manipulate the bytes-read to allow our own read method to coninute - # but keep the window at its current position - self._br = 0 - if hasattr(self._zip, 'status'): - while self._zip.status == zlib.Z_OK: - self.read(mmap.PAGESIZE) - # END scrub-loop custom zlib - else: - # pass in additional pages, until we have unused data - while not self._zip.unused_data and self._cbr != len(self._m): - self.read(mmap.PAGESIZE) - # END scrub-loop default zlib - # END handle stream scrubbing - - # reset bytes read, just to be sure - self._br = self._s - # END handle stream scrubbing - - # unused data ends up in the unconsumed tail, which was removed - # from the count already - return self._cbr - - #} END interface - - def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): - """Allows to reset the stream to restart reading - :raise ValueError: If offset and whence are not 0""" - if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): - raise ValueError("Can only seek to position 0") - # END handle offset - - self._zip = zlib.decompressobj() - self._br = self._cws = self._cwe = self._cbr = 0 - if self._phi: - self._phi = False - del(self._s) # trigger header parsing on first access - # END skip header - - def read(self, size=-1): - if size < 1: - size = self._s - self._br - else: - size = min(size, self._s - self._br) - # END clamp size - - if size == 0: - return str() - # END handle depletion - - - # deplete the buffer, then just continue using the decompress object - # which has an own buffer. We just need this to transparently parse the - # header from the zlib stream - dat = str() - if self._buf: - if self._buflen >= size: - # have enough data - dat = self._buf.read(size) - self._buflen -= size - self._br += size - return dat - else: - dat = self._buf.read() # ouch, duplicates data - size -= self._buflen - self._br += self._buflen - - self._buflen = 0 - self._buf = None - # END handle buffer len - # END handle buffer - - # decompress some data - # Abstract: zlib needs to operate on chunks of our memory map ( which may - # be large ), as it will otherwise and always fill in the 'unconsumed_tail' - # attribute which possible reads our whole map to the end, forcing - # everything to be read from disk even though just a portion was requested. - # As this would be a nogo, we workaround it by passing only chunks of data, - # moving the window into the memory map along as we decompress, which keeps - # the tail smaller than our chunk-size. This causes 'only' the chunk to be - # copied once, and another copy of a part of it when it creates the unconsumed - # tail. We have to use it to hand in the appropriate amount of bytes durin g - # the next read. - tail = self._zip.unconsumed_tail - if tail: - # move the window, make it as large as size demands. For code-clarity, - # we just take the chunk from our map again instead of reusing the unconsumed - # tail. The latter one would safe some memory copying, but we could end up - # with not getting enough data uncompressed, so we had to sort that out as well. - # Now we just assume the worst case, hence the data is uncompressed and the window - # needs to be as large as the uncompressed bytes we want to read. - self._cws = self._cwe - len(tail) - self._cwe = self._cws + size - else: - cws = self._cws - self._cws = self._cwe - self._cwe = cws + size - # END handle tail - - - # if window is too small, make it larger so zip can decompress something - if self._cwe - self._cws < 8: - self._cwe = self._cws + 8 - # END adjust winsize - - # takes a slice, but doesn't copy the data, it says ... - indata = buffer(self._m, self._cws, self._cwe - self._cws) - - # get the actual window end to be sure we don't use it for computations - self._cwe = self._cws + len(indata) - dcompdat = self._zip.decompress(indata, size) - # update the amount of compressed bytes read - # We feed possibly overlapping chunks, which is why the unconsumed tail - # has to be taken into consideration, as well as the unused data - # if we hit the end of the stream - self._cbr += len(indata) - len(self._zip.unconsumed_tail) - self._br += len(dcompdat) - - if dat: - dcompdat = dat + dcompdat - # END prepend our cached data - - # it can happen, depending on the compression, that we get less bytes - # than ordered as it needs the final portion of the data as well. - # Recursively resolve that. - # Note: dcompdat can be empty even though we still appear to have bytes - # to read, if we are called by compressed_bytes_read - it manipulates - # us to empty the stream - if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s: - dcompdat += self.read(size-len(dcompdat)) - # END handle special case - return dcompdat - - + """Reads data in chunks from a memory map and decompresses it. The client sees + only the uncompressed data, respective file-like read calls are handling on-demand + buffered decompression accordingly + + A constraint on the total size of bytes is activated, simulating + a logical file within a possibly larger physical memory area + + To read efficiently, you clearly don't want to read individual bytes, instead, + read a few kilobytes at least. + + **Note:** The chunk-size should be carefully selected as it will involve quite a bit + of string copying due to the way the zlib is implemented. Its very wasteful, + hence we try to find a good tradeoff between allocation time and number of + times we actually allocate. An own zlib implementation would be good here + to better support streamed reading - it would only need to keep the mmap + and decompress it into chunks, thats all ... """ + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', + '_cbr', '_phi') + + max_read_size = 512*1024 # currently unused + + def __init__(self, m, close_on_deletion, size=None): + """Initialize with mmap for stream reading + :param m: must be content data - use new if you have object data and no size""" + self._m = m + self._zip = zlib.decompressobj() + self._buf = None # buffer of decompressed bytes + self._buflen = 0 # length of bytes in buffer + if size is not None: + self._s = size # size of uncompressed data to read in total + self._br = 0 # num uncompressed bytes read + self._cws = 0 # start byte of compression window + self._cwe = 0 # end byte of compression window + self._cbr = 0 # number of compressed bytes read + self._phi = False # is True if we parsed the header info + self._close = close_on_deletion # close the memmap on deletion ? + + def _set_cache_(self, attr): + assert attr == '_s' + # only happens for size, which is a marker to indicate we still + # have to parse the header from the stream + self._parse_header_info() + + def __del__(self): + if self._close: + self._m.close() + # END handle resource freeing + + def _parse_header_info(self): + """If this stream contains object data, parse the header info and skip the + stream to a point where each read will yield object content + + :return: parsed type_string, size""" + # read header + maxb = 512 # should really be enough, cgit uses 8192 I believe + self._s = maxb + hdr = self.read(maxb) + hdrend = hdr.find("\0") + type, size = hdr[:hdrend].split(" ") + size = int(size) + self._s = size + + # adjust internal state to match actual header length that we ignore + # The buffer will be depleted first on future reads + self._br = 0 + hdrend += 1 # count terminating \0 + self._buf = StringIO(hdr[hdrend:]) + self._buflen = len(hdr) - hdrend + + self._phi = True + + return type, size + + #{ Interface + + @classmethod + def new(self, m, close_on_deletion=False): + """Create a new DecompressMemMapReader instance for acting as a read-only stream + This method parses the object header from m and returns the parsed + type and size, as well as the created stream instance. + + :param m: memory map on which to oparate. It must be object data ( header + contents ) + :param close_on_deletion: if True, the memory map will be closed once we are + being deleted""" + inst = DecompressMemMapReader(m, close_on_deletion, 0) + type, size = inst._parse_header_info() + return type, size, inst + + def data(self): + """:return: random access compatible data we are working on""" + return self._m + + def compressed_bytes_read(self): + """ + :return: number of compressed bytes read. This includes the bytes it + took to decompress the header ( if there was one )""" + # ABSTRACT: When decompressing a byte stream, it can be that the first + # x bytes which were requested match the first x bytes in the loosely + # compressed datastream. This is the worst-case assumption that the reader + # does, it assumes that it will get at least X bytes from X compressed bytes + # in call cases. + # The caveat is that the object, according to our known uncompressed size, + # is already complete, but there are still some bytes left in the compressed + # stream that contribute to the amount of compressed bytes. + # How can we know that we are truly done, and have read all bytes we need + # to read ? + # Without help, we cannot know, as we need to obtain the status of the + # decompression. If it is not finished, we need to decompress more data + # until it is finished, to yield the actual number of compressed bytes + # belonging to the decompressed object + # We are using a custom zlib module for this, if its not present, + # we try to put in additional bytes up for decompression if feasible + # and check for the unused_data. + + # Only scrub the stream forward if we are officially done with the + # bytes we were to have. + if self._br == self._s and not self._zip.unused_data: + # manipulate the bytes-read to allow our own read method to coninute + # but keep the window at its current position + self._br = 0 + if hasattr(self._zip, 'status'): + while self._zip.status == zlib.Z_OK: + self.read(mmap.PAGESIZE) + # END scrub-loop custom zlib + else: + # pass in additional pages, until we have unused data + while not self._zip.unused_data and self._cbr != len(self._m): + self.read(mmap.PAGESIZE) + # END scrub-loop default zlib + # END handle stream scrubbing + + # reset bytes read, just to be sure + self._br = self._s + # END handle stream scrubbing + + # unused data ends up in the unconsumed tail, which was removed + # from the count already + return self._cbr + + #} END interface + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Allows to reset the stream to restart reading + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + + self._zip = zlib.decompressobj() + self._br = self._cws = self._cwe = self._cbr = 0 + if self._phi: + self._phi = False + del(self._s) # trigger header parsing on first access + # END skip header + + def read(self, size=-1): + if size < 1: + size = self._s - self._br + else: + size = min(size, self._s - self._br) + # END clamp size + + if size == 0: + return str() + # END handle depletion + + + # deplete the buffer, then just continue using the decompress object + # which has an own buffer. We just need this to transparently parse the + # header from the zlib stream + dat = str() + if self._buf: + if self._buflen >= size: + # have enough data + dat = self._buf.read(size) + self._buflen -= size + self._br += size + return dat + else: + dat = self._buf.read() # ouch, duplicates data + size -= self._buflen + self._br += self._buflen + + self._buflen = 0 + self._buf = None + # END handle buffer len + # END handle buffer + + # decompress some data + # Abstract: zlib needs to operate on chunks of our memory map ( which may + # be large ), as it will otherwise and always fill in the 'unconsumed_tail' + # attribute which possible reads our whole map to the end, forcing + # everything to be read from disk even though just a portion was requested. + # As this would be a nogo, we workaround it by passing only chunks of data, + # moving the window into the memory map along as we decompress, which keeps + # the tail smaller than our chunk-size. This causes 'only' the chunk to be + # copied once, and another copy of a part of it when it creates the unconsumed + # tail. We have to use it to hand in the appropriate amount of bytes durin g + # the next read. + tail = self._zip.unconsumed_tail + if tail: + # move the window, make it as large as size demands. For code-clarity, + # we just take the chunk from our map again instead of reusing the unconsumed + # tail. The latter one would safe some memory copying, but we could end up + # with not getting enough data uncompressed, so we had to sort that out as well. + # Now we just assume the worst case, hence the data is uncompressed and the window + # needs to be as large as the uncompressed bytes we want to read. + self._cws = self._cwe - len(tail) + self._cwe = self._cws + size + else: + cws = self._cws + self._cws = self._cwe + self._cwe = cws + size + # END handle tail + + + # if window is too small, make it larger so zip can decompress something + if self._cwe - self._cws < 8: + self._cwe = self._cws + 8 + # END adjust winsize + + # takes a slice, but doesn't copy the data, it says ... + indata = buffer(self._m, self._cws, self._cwe - self._cws) + + # get the actual window end to be sure we don't use it for computations + self._cwe = self._cws + len(indata) + dcompdat = self._zip.decompress(indata, size) + # update the amount of compressed bytes read + # We feed possibly overlapping chunks, which is why the unconsumed tail + # has to be taken into consideration, as well as the unused data + # if we hit the end of the stream + self._cbr += len(indata) - len(self._zip.unconsumed_tail) + self._br += len(dcompdat) + + if dat: + dcompdat = dat + dcompdat + # END prepend our cached data + + # it can happen, depending on the compression, that we get less bytes + # than ordered as it needs the final portion of the data as well. + # Recursively resolve that. + # Note: dcompdat can be empty even though we still appear to have bytes + # to read, if we are called by compressed_bytes_read - it manipulates + # us to empty the stream + if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s: + dcompdat += self.read(size-len(dcompdat)) + # END handle special case + return dcompdat + + class DeltaApplyReader(LazyMixin): - """A reader which dynamically applies pack deltas to a base object, keeping the - memory demands to a minimum. - - The size of the final object is only obtainable once all deltas have been - applied, unless it is retrieved from a pack index. - - The uncompressed Delta has the following layout (MSB being a most significant - bit encoded dynamic size): - - * MSB Source Size - the size of the base against which the delta was created - * MSB Target Size - the size of the resulting data after the delta was applied - * A list of one byte commands (cmd) which are followed by a specific protocol: - - * cmd & 0x80 - copy delta_data[offset:offset+size] - - * Followed by an encoded offset into the delta data - * Followed by an encoded size of the chunk to copy - - * cmd & 0x7f - insert - - * insert cmd bytes from the delta buffer into the output stream - - * cmd == 0 - invalid operation ( or error in delta stream ) - """ - __slots__ = ( - "_bstream", # base stream to which to apply the deltas - "_dstreams", # tuple of delta stream readers - "_mm_target", # memory map of the delta-applied data - "_size", # actual number of bytes in _mm_target - "_br" # number of bytes read - ) - - #{ Configuration - k_max_memory_move = 250*1000*1000 - #} END configuration - - def __init__(self, stream_list): - """Initialize this instance with a list of streams, the first stream being - the delta to apply on top of all following deltas, the last stream being the - base object onto which to apply the deltas""" - assert len(stream_list) > 1, "Need at least one delta and one base stream" - - self._bstream = stream_list[-1] - self._dstreams = tuple(stream_list[:-1]) - self._br = 0 - - def _set_cache_too_slow_without_c(self, attr): - # the direct algorithm is fastest and most direct if there is only one - # delta. Also, the extra overhead might not be worth it for items smaller - # than X - definitely the case in python, every function call costs - # huge amounts of time - # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: - if len(self._dstreams) == 1: - return self._set_cache_brute_(attr) - - # Aggregate all deltas into one delta in reverse order. Hence we take - # the last delta, and reverse-merge its ancestor delta, until we receive - # the final delta data stream. - # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) - dcl = connect_deltas(self._dstreams) - - # call len directly, as the (optional) c version doesn't implement the sequence - # protocol - if dcl.rbound() == 0: - self._size = 0 - self._mm_target = allocate_memory(0) - return - # END handle empty list - - self._size = dcl.rbound() - self._mm_target = allocate_memory(self._size) - - bbuf = allocate_memory(self._bstream.size) - stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) - - # APPLY CHUNKS - write = self._mm_target.write - dcl.apply(bbuf, write) - - self._mm_target.seek(0) - - def _set_cache_brute_(self, attr): - """If we are here, we apply the actual deltas""" - - # TODO: There should be a special case if there is only one stream - # Then the default-git algorithm should perform a tad faster, as the - # delta is not peaked into, causing less overhead. - buffer_info_list = list() - max_target_size = 0 - for dstream in self._dstreams: - buf = dstream.read(512) # read the header information + X - offset, src_size = msb_size(buf) - offset, target_size = msb_size(buf, offset) - buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) - max_target_size = max(max_target_size, target_size) - # END for each delta stream - - # sanity check - the first delta to apply should have the same source - # size as our actual base stream - base_size = self._bstream.size - target_size = max_target_size - - # if we have more than 1 delta to apply, we will swap buffers, hence we must - # assure that all buffers we use are large enough to hold all the results - if len(self._dstreams) > 1: - base_size = target_size = max(base_size, max_target_size) - # END adjust buffer sizes - - - # Allocate private memory map big enough to hold the first base buffer - # We need random access to it - bbuf = allocate_memory(base_size) - stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) - - # allocate memory map large enough for the largest (intermediate) target - # We will use it as scratch space for all delta ops. If the final - # target buffer is smaller than our allocated space, we just use parts - # of it upon return. - tbuf = allocate_memory(target_size) - - # for each delta to apply, memory map the decompressed delta and - # work on the op-codes to reconstruct everything. - # For the actual copying, we use a seek and write pattern of buffer - # slices. - final_target_size = None - for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): - # allocate a buffer to hold all delta data - fill in the data for - # fast access. We do this as we know that reading individual bytes - # from our stream would be slower than necessary ( although possible ) - # The dbuf buffer contains commands after the first two MSB sizes, the - # offset specifies the amount of bytes read to get the sizes. - ddata = allocate_memory(dstream.size - offset) - ddata.write(dbuf) - # read the rest from the stream. The size we give is larger than necessary - stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) - - ####################################################################### - if 'c_apply_delta' in globals(): - c_apply_delta(bbuf, ddata, tbuf); - else: - apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) - ####################################################################### - - # finally, swap out source and target buffers. The target is now the - # base for the next delta to apply - bbuf, tbuf = tbuf, bbuf - bbuf.seek(0) - tbuf.seek(0) - final_target_size = target_size - # END for each delta to apply - - # its already seeked to 0, constrain it to the actual size - # NOTE: in the end of the loop, it swaps buffers, hence our target buffer - # is not tbuf, but bbuf ! - self._mm_target = bbuf - self._size = final_target_size - - - #{ Configuration - if not has_perf_mod: - _set_cache_ = _set_cache_brute_ - else: - _set_cache_ = _set_cache_too_slow_without_c - - #} END configuration - - def read(self, count=0): - bl = self._size - self._br # bytes left - if count < 1 or count > bl: - count = bl - # NOTE: we could check for certain size limits, and possibly - # return buffers instead of strings to prevent byte copying - data = self._mm_target.read(count) - self._br += len(data) - return data - - def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): - """Allows to reset the stream to restart reading - - :raise ValueError: If offset and whence are not 0""" - if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): - raise ValueError("Can only seek to position 0") - # END handle offset - self._br = 0 - self._mm_target.seek(0) - - #{ Interface - - @classmethod - def new(cls, stream_list): - """ - Convert the given list of streams into a stream which resolves deltas - when reading from it. - - :param stream_list: two or more stream objects, first stream is a Delta - to the object that you want to resolve, followed by N additional delta - streams. The list's last stream must be a non-delta stream. - - :return: Non-Delta OPackStream object whose stream can be used to obtain - the decompressed resolved data - :raise ValueError: if the stream list cannot be handled""" - if len(stream_list) < 2: - raise ValueError("Need at least two streams") - # END single object special handling - - if stream_list[-1].type_id in delta_types: - raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) - # END check stream - - return cls(stream_list) - - #} END interface - - - #{ OInfo like Interface - - @property - def type(self): - return self._bstream.type - - @property - def type_id(self): - return self._bstream.type_id - - @property - def size(self): - """:return: number of uncompressed bytes in the stream""" - return self._size - - #} END oinfo like interface - - + """A reader which dynamically applies pack deltas to a base object, keeping the + memory demands to a minimum. + + The size of the final object is only obtainable once all deltas have been + applied, unless it is retrieved from a pack index. + + The uncompressed Delta has the following layout (MSB being a most significant + bit encoded dynamic size): + + * MSB Source Size - the size of the base against which the delta was created + * MSB Target Size - the size of the resulting data after the delta was applied + * A list of one byte commands (cmd) which are followed by a specific protocol: + + * cmd & 0x80 - copy delta_data[offset:offset+size] + + * Followed by an encoded offset into the delta data + * Followed by an encoded size of the chunk to copy + + * cmd & 0x7f - insert + + * insert cmd bytes from the delta buffer into the output stream + + * cmd == 0 - invalid operation ( or error in delta stream ) + """ + __slots__ = ( + "_bstream", # base stream to which to apply the deltas + "_dstreams", # tuple of delta stream readers + "_mm_target", # memory map of the delta-applied data + "_size", # actual number of bytes in _mm_target + "_br" # number of bytes read + ) + + #{ Configuration + k_max_memory_move = 250*1000*1000 + #} END configuration + + def __init__(self, stream_list): + """Initialize this instance with a list of streams, the first stream being + the delta to apply on top of all following deltas, the last stream being the + base object onto which to apply the deltas""" + assert len(stream_list) > 1, "Need at least one delta and one base stream" + + self._bstream = stream_list[-1] + self._dstreams = tuple(stream_list[:-1]) + self._br = 0 + + def _set_cache_too_slow_without_c(self, attr): + # the direct algorithm is fastest and most direct if there is only one + # delta. Also, the extra overhead might not be worth it for items smaller + # than X - definitely the case in python, every function call costs + # huge amounts of time + # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: + if len(self._dstreams) == 1: + return self._set_cache_brute_(attr) + + # Aggregate all deltas into one delta in reverse order. Hence we take + # the last delta, and reverse-merge its ancestor delta, until we receive + # the final delta data stream. + # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) + dcl = connect_deltas(self._dstreams) + + # call len directly, as the (optional) c version doesn't implement the sequence + # protocol + if dcl.rbound() == 0: + self._size = 0 + self._mm_target = allocate_memory(0) + return + # END handle empty list + + self._size = dcl.rbound() + self._mm_target = allocate_memory(self._size) + + bbuf = allocate_memory(self._bstream.size) + stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) + + # APPLY CHUNKS + write = self._mm_target.write + dcl.apply(bbuf, write) + + self._mm_target.seek(0) + + def _set_cache_brute_(self, attr): + """If we are here, we apply the actual deltas""" + + # TODO: There should be a special case if there is only one stream + # Then the default-git algorithm should perform a tad faster, as the + # delta is not peaked into, causing less overhead. + buffer_info_list = list() + max_target_size = 0 + for dstream in self._dstreams: + buf = dstream.read(512) # read the header information + X + offset, src_size = msb_size(buf) + offset, target_size = msb_size(buf, offset) + buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) + max_target_size = max(max_target_size, target_size) + # END for each delta stream + + # sanity check - the first delta to apply should have the same source + # size as our actual base stream + base_size = self._bstream.size + target_size = max_target_size + + # if we have more than 1 delta to apply, we will swap buffers, hence we must + # assure that all buffers we use are large enough to hold all the results + if len(self._dstreams) > 1: + base_size = target_size = max(base_size, max_target_size) + # END adjust buffer sizes + + + # Allocate private memory map big enough to hold the first base buffer + # We need random access to it + bbuf = allocate_memory(base_size) + stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) + + # allocate memory map large enough for the largest (intermediate) target + # We will use it as scratch space for all delta ops. If the final + # target buffer is smaller than our allocated space, we just use parts + # of it upon return. + tbuf = allocate_memory(target_size) + + # for each delta to apply, memory map the decompressed delta and + # work on the op-codes to reconstruct everything. + # For the actual copying, we use a seek and write pattern of buffer + # slices. + final_target_size = None + for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): + # allocate a buffer to hold all delta data - fill in the data for + # fast access. We do this as we know that reading individual bytes + # from our stream would be slower than necessary ( although possible ) + # The dbuf buffer contains commands after the first two MSB sizes, the + # offset specifies the amount of bytes read to get the sizes. + ddata = allocate_memory(dstream.size - offset) + ddata.write(dbuf) + # read the rest from the stream. The size we give is larger than necessary + stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) + + ####################################################################### + if 'c_apply_delta' in globals(): + c_apply_delta(bbuf, ddata, tbuf); + else: + apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) + ####################################################################### + + # finally, swap out source and target buffers. The target is now the + # base for the next delta to apply + bbuf, tbuf = tbuf, bbuf + bbuf.seek(0) + tbuf.seek(0) + final_target_size = target_size + # END for each delta to apply + + # its already seeked to 0, constrain it to the actual size + # NOTE: in the end of the loop, it swaps buffers, hence our target buffer + # is not tbuf, but bbuf ! + self._mm_target = bbuf + self._size = final_target_size + + + #{ Configuration + if not has_perf_mod: + _set_cache_ = _set_cache_brute_ + else: + _set_cache_ = _set_cache_too_slow_without_c + + #} END configuration + + def read(self, count=0): + bl = self._size - self._br # bytes left + if count < 1 or count > bl: + count = bl + # NOTE: we could check for certain size limits, and possibly + # return buffers instead of strings to prevent byte copying + data = self._mm_target.read(count) + self._br += len(data) + return data + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Allows to reset the stream to restart reading + + :raise ValueError: If offset and whence are not 0""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + self._br = 0 + self._mm_target.seek(0) + + #{ Interface + + @classmethod + def new(cls, stream_list): + """ + Convert the given list of streams into a stream which resolves deltas + when reading from it. + + :param stream_list: two or more stream objects, first stream is a Delta + to the object that you want to resolve, followed by N additional delta + streams. The list's last stream must be a non-delta stream. + + :return: Non-Delta OPackStream object whose stream can be used to obtain + the decompressed resolved data + :raise ValueError: if the stream list cannot be handled""" + if len(stream_list) < 2: + raise ValueError("Need at least two streams") + # END single object special handling + + if stream_list[-1].type_id in delta_types: + raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) + # END check stream + + return cls(stream_list) + + #} END interface + + + #{ OInfo like Interface + + @property + def type(self): + return self._bstream.type + + @property + def type_id(self): + return self._bstream.type_id + + @property + def size(self): + """:return: number of uncompressed bytes in the stream""" + return self._size + + #} END oinfo like interface + + #} END RO streams #{ W Streams class Sha1Writer(object): - """Simple stream writer which produces a sha whenever you like as it degests - everything it is supposed to write""" - __slots__ = "sha1" - - def __init__(self): - self.sha1 = make_sha() - - #{ Stream Interface - - def write(self, data): - """:raise IOError: If not all bytes could be written - :return: lenght of incoming data""" - self.sha1.update(data) - return len(data) - - # END stream interface - - #{ Interface - - def sha(self, as_hex = False): - """:return: sha so far - :param as_hex: if True, sha will be hex-encoded, binary otherwise""" - if as_hex: - return self.sha1.hexdigest() - return self.sha1.digest() - - #} END interface + """Simple stream writer which produces a sha whenever you like as it degests + everything it is supposed to write""" + __slots__ = "sha1" + + def __init__(self): + self.sha1 = make_sha() + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: lenght of incoming data""" + self.sha1.update(data) + return len(data) + + # END stream interface + + #{ Interface + + def sha(self, as_hex = False): + """:return: sha so far + :param as_hex: if True, sha will be hex-encoded, binary otherwise""" + if as_hex: + return self.sha1.hexdigest() + return self.sha1.digest() + + #} END interface class FlexibleSha1Writer(Sha1Writer): - """Writer producing a sha1 while passing on the written bytes to the given - write function""" - __slots__ = 'writer' - - def __init__(self, writer): - Sha1Writer.__init__(self) - self.writer = writer - - def write(self, data): - Sha1Writer.write(self, data) - self.writer(data) + """Writer producing a sha1 while passing on the written bytes to the given + write function""" + __slots__ = 'writer' + + def __init__(self, writer): + Sha1Writer.__init__(self) + self.writer = writer + + def write(self, data): + Sha1Writer.write(self, data) + self.writer(data) class ZippedStoreShaWriter(Sha1Writer): - """Remembers everything someone writes to it and generates a sha""" - __slots__ = ('buf', 'zip') - def __init__(self): - Sha1Writer.__init__(self) - self.buf = StringIO() - self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) - - def __getattr__(self, attr): - return getattr(self.buf, attr) - - def write(self, data): - alen = Sha1Writer.write(self, data) - self.buf.write(self.zip.compress(data)) - return alen - - def close(self): - self.buf.write(self.zip.flush()) - - def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): - """Seeking currently only supports to rewind written data - Multiple writes are not supported""" - if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): - raise ValueError("Can only seek to position 0") - # END handle offset - self.buf.seek(0) - - def getvalue(self): - """:return: string value from the current stream position to the end""" - return self.buf.getvalue() + """Remembers everything someone writes to it and generates a sha""" + __slots__ = ('buf', 'zip') + def __init__(self): + Sha1Writer.__init__(self) + self.buf = StringIO() + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) + + def __getattr__(self, attr): + return getattr(self.buf, attr) + + def write(self, data): + alen = Sha1Writer.write(self, data) + self.buf.write(self.zip.compress(data)) + return alen + + def close(self): + self.buf.write(self.zip.flush()) + + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): + """Seeking currently only supports to rewind written data + Multiple writes are not supported""" + if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): + raise ValueError("Can only seek to position 0") + # END handle offset + self.buf.seek(0) + + def getvalue(self): + """:return: string value from the current stream position to the end""" + return self.buf.getvalue() class FDCompressedSha1Writer(Sha1Writer): - """Digests data written to it, making the sha available, then compress the - data and write it to the file descriptor - - **Note:** operates on raw file descriptors - **Note:** for this to work, you have to use the close-method of this instance""" - __slots__ = ("fd", "sha1", "zip") - - # default exception - exc = IOError("Failed to write all bytes to filedescriptor") - - def __init__(self, fd): - super(FDCompressedSha1Writer, self).__init__() - self.fd = fd - self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) - - #{ Stream Interface - - def write(self, data): - """:raise IOError: If not all bytes could be written - :return: lenght of incoming data""" - self.sha1.update(data) - cdata = self.zip.compress(data) - bytes_written = write(self.fd, cdata) - if bytes_written != len(cdata): - raise self.exc - return len(data) - - def close(self): - remainder = self.zip.flush() - if write(self.fd, remainder) != len(remainder): - raise self.exc - return close(self.fd) - - #} END stream interface + """Digests data written to it, making the sha available, then compress the + data and write it to the file descriptor + + **Note:** operates on raw file descriptors + **Note:** for this to work, you have to use the close-method of this instance""" + __slots__ = ("fd", "sha1", "zip") + + # default exception + exc = IOError("Failed to write all bytes to filedescriptor") + + def __init__(self, fd): + super(FDCompressedSha1Writer, self).__init__() + self.fd = fd + self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) + + #{ Stream Interface + + def write(self, data): + """:raise IOError: If not all bytes could be written + :return: lenght of incoming data""" + self.sha1.update(data) + cdata = self.zip.compress(data) + bytes_written = write(self.fd, cdata) + if bytes_written != len(cdata): + raise self.exc + return len(data) + + def close(self): + remainder = self.zip.flush() + if write(self.fd, remainder) != len(remainder): + raise self.exc + return close(self.fd) + + #} END stream interface class FDStream(object): - """A simple wrapper providing the most basic functions on a file descriptor - with the fileobject interface. Cannot use os.fdopen as the resulting stream - takes ownership""" - __slots__ = ("_fd", '_pos') - def __init__(self, fd): - self._fd = fd - self._pos = 0 - - def write(self, data): - self._pos += len(data) - os.write(self._fd, data) - - def read(self, count=0): - if count == 0: - count = os.path.getsize(self._filepath) - # END handle read everything - - bytes = os.read(self._fd, count) - self._pos += len(bytes) - return bytes - - def fileno(self): - return self._fd - - def tell(self): - return self._pos - - def close(self): - close(self._fd) + """A simple wrapper providing the most basic functions on a file descriptor + with the fileobject interface. Cannot use os.fdopen as the resulting stream + takes ownership""" + __slots__ = ("_fd", '_pos') + def __init__(self, fd): + self._fd = fd + self._pos = 0 + + def write(self, data): + self._pos += len(data) + os.write(self._fd, data) + + def read(self, count=0): + if count == 0: + count = os.path.getsize(self._filepath) + # END handle read everything + + bytes = os.read(self._fd, count) + self._pos += len(bytes) + return bytes + + def fileno(self): + return self._fd + + def tell(self): + return self._pos + + def close(self): + close(self._fd) class NullStream(object): - """A stream that does nothing but providing a stream interface. - Use it like /dev/null""" - __slots__ = tuple() - - def read(self, size=0): - return '' - - def close(self): - pass - - def write(self, data): - return len(data) + """A stream that does nothing but providing a stream interface. + Use it like /dev/null""" + __slots__ = tuple() + + def read(self, size=0): + return '' + + def close(self): + pass + + def write(self, data): + return len(data) #} END W streams diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index 760f531be..f8059447f 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -7,10 +7,10 @@ #{ Initialization def _init_pool(): - """Assure the pool is actually threaded""" - size = 2 - print "Setting ThreadPool to %i" % size - gitdb.util.pool.set_size(size) + """Assure the pool is actually threaded""" + size = 2 + print "Setting ThreadPool to %i" % size + gitdb.util.pool.set_size(size) #} END initialization diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 4af4483c7..62614ee5c 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -4,21 +4,21 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Base classes for object db testing""" from gitdb.test.lib import ( - with_rw_directory, - with_packs_rw, - ZippedStoreShaWriter, - fixture_path, - TestBase - ) + with_rw_directory, + with_packs_rw, + ZippedStoreShaWriter, + fixture_path, + TestBase + ) from gitdb.stream import Sha1Writer from gitdb.base import ( - IStream, - OStream, - OInfo - ) - + IStream, + OStream, + OInfo + ) + from gitdb.exc import BadObject from gitdb.typ import str_blob_type @@ -28,181 +28,181 @@ __all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') - + class TestDBBase(TestBase): - """Base class providing testing routines on databases""" - - # data - two_lines = "1234\nhello world" - all_data = (two_lines, ) - - def _assert_object_writing_simple(self, db): - # write a bunch of objects and query their streams and info - null_objs = db.size() - ni = 250 - for i in xrange(ni): - data = pack(">L", i) - istream = IStream(str_blob_type, len(data), StringIO(data)) - new_istream = db.store(istream) - assert new_istream is istream - assert db.has_object(istream.binsha) - - info = db.info(istream.binsha) - assert isinstance(info, OInfo) - assert info.type == istream.type and info.size == istream.size - - stream = db.stream(istream.binsha) - assert isinstance(stream, OStream) - assert stream.binsha == info.binsha and stream.type == info.type - assert stream.read() == data - # END for each item - - assert db.size() == null_objs + ni - shas = list(db.sha_iter()) - assert len(shas) == db.size() - assert len(shas[0]) == 20 - - - def _assert_object_writing(self, db): - """General tests to verify object writing, compatible to ObjectDBW - **Note:** requires write access to the database""" - # start in 'dry-run' mode, using a simple sha1 writer - ostreams = (ZippedStoreShaWriter, None) - for ostreamcls in ostreams: - for data in self.all_data: - dry_run = ostreamcls is not None - ostream = None - if ostreamcls is not None: - ostream = ostreamcls() - assert isinstance(ostream, Sha1Writer) - # END create ostream - - prev_ostream = db.set_ostream(ostream) - assert type(prev_ostream) in ostreams or prev_ostream in ostreams - - istream = IStream(str_blob_type, len(data), StringIO(data)) - - # store returns same istream instance, with new sha set - my_istream = db.store(istream) - sha = istream.binsha - assert my_istream is istream - assert db.has_object(sha) != dry_run - assert len(sha) == 20 - - # verify data - the slow way, we want to run code - if not dry_run: - info = db.info(sha) - assert str_blob_type == info.type - assert info.size == len(data) - - ostream = db.stream(sha) - assert ostream.read() == data - assert ostream.type == str_blob_type - assert ostream.size == len(data) - else: - self.failUnlessRaises(BadObject, db.info, sha) - self.failUnlessRaises(BadObject, db.stream, sha) - - # DIRECT STREAM COPY - # our data hase been written in object format to the StringIO - # we pasesd as output stream. No physical database representation - # was created. - # Test direct stream copy of object streams, the result must be - # identical to what we fed in - ostream.seek(0) - istream.stream = ostream - assert istream.binsha is not None - prev_sha = istream.binsha - - db.set_ostream(ZippedStoreShaWriter()) - db.store(istream) - assert istream.binsha == prev_sha - new_ostream = db.ostream() - - # note: only works as long our store write uses the same compression - # level, which is zip_best - assert ostream.getvalue() == new_ostream.getvalue() - # END for each data set - # END for each dry_run mode - - def _assert_object_writing_async(self, db): - """Test generic object writing using asynchronous access""" - ni = 5000 - def istream_generator(offset=0, ni=ni): - for data_src in xrange(ni): - data = str(data_src + offset) - yield IStream(str_blob_type, len(data), StringIO(data)) - # END for each item - # END generator utility - - # for now, we are very trusty here as we expect it to work if it worked - # in the single-stream case - - # write objects - reader = IteratorReader(istream_generator()) - istream_reader = db.store_async(reader) - istreams = istream_reader.read() # read all - assert istream_reader.task().error() is None - assert len(istreams) == ni - - for stream in istreams: - assert stream.error is None - assert len(stream.binsha) == 20 - assert isinstance(stream, IStream) - # END assert each stream - - # test has-object-async - we must have all previously added ones - reader = IteratorReader( istream.binsha for istream in istreams ) - hasobject_reader = db.has_object_async(reader) - count = 0 - for sha, has_object in hasobject_reader: - assert has_object - count += 1 - # END for each sha - assert count == ni - - # read the objects we have just written - reader = IteratorReader( istream.binsha for istream in istreams ) - ostream_reader = db.stream_async(reader) - - # read items individually to prevent hitting possible sys-limits - count = 0 - for ostream in ostream_reader: - assert isinstance(ostream, OStream) - count += 1 - # END for each ostream - assert ostream_reader.task().error() is None - assert count == ni - - # get info about our items - reader = IteratorReader( istream.binsha for istream in istreams ) - info_reader = db.info_async(reader) - - count = 0 - for oinfo in info_reader: - assert isinstance(oinfo, OInfo) - count += 1 - # END for each oinfo instance - assert count == ni - - - # combined read-write using a converter - # add 2500 items, and obtain their output streams - nni = 2500 - reader = IteratorReader(istream_generator(offset=ni, ni=nni)) - istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] - - istream_reader = db.store_async(reader) - istream_reader.set_post_cb(istream_to_sha) - - ostream_reader = db.stream_async(istream_reader) - - count = 0 - # read it individually, otherwise we might run into the ulimit - for ostream in ostream_reader: - assert isinstance(ostream, OStream) - count += 1 - # END for each ostream - assert count == nni - - + """Base class providing testing routines on databases""" + + # data + two_lines = "1234\nhello world" + all_data = (two_lines, ) + + def _assert_object_writing_simple(self, db): + # write a bunch of objects and query their streams and info + null_objs = db.size() + ni = 250 + for i in xrange(ni): + data = pack(">L", i) + istream = IStream(str_blob_type, len(data), StringIO(data)) + new_istream = db.store(istream) + assert new_istream is istream + assert db.has_object(istream.binsha) + + info = db.info(istream.binsha) + assert isinstance(info, OInfo) + assert info.type == istream.type and info.size == istream.size + + stream = db.stream(istream.binsha) + assert isinstance(stream, OStream) + assert stream.binsha == info.binsha and stream.type == info.type + assert stream.read() == data + # END for each item + + assert db.size() == null_objs + ni + shas = list(db.sha_iter()) + assert len(shas) == db.size() + assert len(shas[0]) == 20 + + + def _assert_object_writing(self, db): + """General tests to verify object writing, compatible to ObjectDBW + **Note:** requires write access to the database""" + # start in 'dry-run' mode, using a simple sha1 writer + ostreams = (ZippedStoreShaWriter, None) + for ostreamcls in ostreams: + for data in self.all_data: + dry_run = ostreamcls is not None + ostream = None + if ostreamcls is not None: + ostream = ostreamcls() + assert isinstance(ostream, Sha1Writer) + # END create ostream + + prev_ostream = db.set_ostream(ostream) + assert type(prev_ostream) in ostreams or prev_ostream in ostreams + + istream = IStream(str_blob_type, len(data), StringIO(data)) + + # store returns same istream instance, with new sha set + my_istream = db.store(istream) + sha = istream.binsha + assert my_istream is istream + assert db.has_object(sha) != dry_run + assert len(sha) == 20 + + # verify data - the slow way, we want to run code + if not dry_run: + info = db.info(sha) + assert str_blob_type == info.type + assert info.size == len(data) + + ostream = db.stream(sha) + assert ostream.read() == data + assert ostream.type == str_blob_type + assert ostream.size == len(data) + else: + self.failUnlessRaises(BadObject, db.info, sha) + self.failUnlessRaises(BadObject, db.stream, sha) + + # DIRECT STREAM COPY + # our data hase been written in object format to the StringIO + # we pasesd as output stream. No physical database representation + # was created. + # Test direct stream copy of object streams, the result must be + # identical to what we fed in + ostream.seek(0) + istream.stream = ostream + assert istream.binsha is not None + prev_sha = istream.binsha + + db.set_ostream(ZippedStoreShaWriter()) + db.store(istream) + assert istream.binsha == prev_sha + new_ostream = db.ostream() + + # note: only works as long our store write uses the same compression + # level, which is zip_best + assert ostream.getvalue() == new_ostream.getvalue() + # END for each data set + # END for each dry_run mode + + def _assert_object_writing_async(self, db): + """Test generic object writing using asynchronous access""" + ni = 5000 + def istream_generator(offset=0, ni=ni): + for data_src in xrange(ni): + data = str(data_src + offset) + yield IStream(str_blob_type, len(data), StringIO(data)) + # END for each item + # END generator utility + + # for now, we are very trusty here as we expect it to work if it worked + # in the single-stream case + + # write objects + reader = IteratorReader(istream_generator()) + istream_reader = db.store_async(reader) + istreams = istream_reader.read() # read all + assert istream_reader.task().error() is None + assert len(istreams) == ni + + for stream in istreams: + assert stream.error is None + assert len(stream.binsha) == 20 + assert isinstance(stream, IStream) + # END assert each stream + + # test has-object-async - we must have all previously added ones + reader = IteratorReader( istream.binsha for istream in istreams ) + hasobject_reader = db.has_object_async(reader) + count = 0 + for sha, has_object in hasobject_reader: + assert has_object + count += 1 + # END for each sha + assert count == ni + + # read the objects we have just written + reader = IteratorReader( istream.binsha for istream in istreams ) + ostream_reader = db.stream_async(reader) + + # read items individually to prevent hitting possible sys-limits + count = 0 + for ostream in ostream_reader: + assert isinstance(ostream, OStream) + count += 1 + # END for each ostream + assert ostream_reader.task().error() is None + assert count == ni + + # get info about our items + reader = IteratorReader( istream.binsha for istream in istreams ) + info_reader = db.info_async(reader) + + count = 0 + for oinfo in info_reader: + assert isinstance(oinfo, OInfo) + count += 1 + # END for each oinfo instance + assert count == ni + + + # combined read-write using a converter + # add 2500 items, and obtain their output streams + nni = 2500 + reader = IteratorReader(istream_generator(offset=ni, ni=nni)) + istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] + + istream_reader = db.store_async(reader) + istream_reader.set_post_cb(istream_to_sha) + + ostream_reader = db.stream_async(istream_reader) + + count = 0 + # read it individually, otherwise we might run into the ulimit + for ostream in ostream_reader: + assert isinstance(ostream, OStream) + count += 1 + # END for each ostream + assert count == nni + + diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 310116351..1ef577aa3 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -7,41 +7,41 @@ from gitdb.db import GitDB from gitdb.base import OStream, OInfo from gitdb.util import hex_to_bin, bin_to_hex - + class TestGitDB(TestDBBase): - - def test_reading(self): - gdb = GitDB(fixture_path('../../../.git/objects')) - - # we have packs and loose objects, alternates doesn't necessarily exist - assert 1 < len(gdb.databases()) < 4 - - # access should be possible - gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") - assert isinstance(gdb.info(gitdb_sha), OInfo) - assert isinstance(gdb.stream(gitdb_sha), OStream) - assert gdb.size() > 200 - sha_list = list(gdb.sha_iter()) - assert len(sha_list) == gdb.size() - - - # This is actually a test for compound functionality, but it doesn't - # have a separate test module - # test partial shas - # this one as uneven and quite short - assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") - - # mix even/uneven hexshas - for i, binsha in enumerate(sha_list): - assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha - # END for each sha - - self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") - - @with_rw_directory - def test_writing(self, path): - gdb = GitDB(path) - - # its possible to write objects - self._assert_object_writing(gdb) - self._assert_object_writing_async(gdb) + + def test_reading(self): + gdb = GitDB(fixture_path('../../../.git/objects')) + + # we have packs and loose objects, alternates doesn't necessarily exist + assert 1 < len(gdb.databases()) < 4 + + # access should be possible + gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") + assert isinstance(gdb.info(gitdb_sha), OInfo) + assert isinstance(gdb.stream(gitdb_sha), OStream) + assert gdb.size() > 200 + sha_list = list(gdb.sha_iter()) + assert len(sha_list) == gdb.size() + + + # This is actually a test for compound functionality, but it doesn't + # have a separate test module + # test partial shas + # this one as uneven and quite short + assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") + + # mix even/uneven hexshas + for i, binsha in enumerate(sha_list): + assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha + # END for each sha + + self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") + + @with_rw_directory + def test_writing(self, path): + gdb = GitDB(path) + + # its possible to write objects + self._assert_object_writing(gdb) + self._assert_object_writing_async(gdb) diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index ee2d78d08..d7e1d01b0 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -6,29 +6,29 @@ from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex - + class TestLooseDB(TestDBBase): - - @with_rw_directory - def test_basics(self, path): - ldb = LooseObjectDB(path) - - # write data - self._assert_object_writing(ldb) - self._assert_object_writing_async(ldb) - - # verify sha iteration and size - shas = list(ldb.sha_iter()) - assert shas and len(shas[0]) == 20 - - assert len(shas) == ldb.size() - - # verify find short object - long_sha = bin_to_hex(shas[-1]) - for short_sha in (long_sha[:20], long_sha[:5]): - assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha - # END for each sha - - self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') - # raises if no object could be foudn - + + @with_rw_directory + def test_basics(self, path): + ldb = LooseObjectDB(path) + + # write data + self._assert_object_writing(ldb) + self._assert_object_writing_async(ldb) + + # verify sha iteration and size + shas = list(ldb.sha_iter()) + assert shas and len(shas[0]) == 20 + + assert len(shas) == ldb.size() + + # verify find short object + long_sha = bin_to_hex(shas[-1]) + for short_sha in (long_sha[:20], long_sha[:5]): + assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha + # END for each sha + + self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') + # raises if no object could be foudn + diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index 188cb0a93..df428e2b7 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -4,27 +4,27 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import ( - MemoryDB, - LooseObjectDB - ) - + MemoryDB, + LooseObjectDB + ) + class TestMemoryDB(TestDBBase): - - @with_rw_directory - def test_writing(self, path): - mdb = MemoryDB() - - # write data - self._assert_object_writing_simple(mdb) - - # test stream copy - ldb = LooseObjectDB(path) - assert ldb.size() == 0 - num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) - assert num_streams_copied == mdb.size() - - assert ldb.size() == mdb.size() - for sha in mdb.sha_iter(): - assert ldb.has_object(sha) - assert ldb.stream(sha).read() == mdb.stream(sha).read() - # END verify objects where copied and are equal + + @with_rw_directory + def test_writing(self, path): + mdb = MemoryDB() + + # write data + self._assert_object_writing_simple(mdb) + + # test stream copy + ldb = LooseObjectDB(path) + assert ldb.size() == 0 + num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) + assert num_streams_copied == mdb.size() + + assert ldb.size() == mdb.size() + for sha in mdb.sha_iter(): + assert ldb.has_object(sha) + assert ldb.stream(sha).read() == mdb.stream(sha).read() + # END verify objects where copied and are equal diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index e8ba6f8fc..f4cb5bbc6 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -12,62 +12,62 @@ import random class TestPackDB(TestDBBase): - - @with_rw_directory - @with_packs_rw - def test_writing(self, path): - pdb = PackedDB(path) - - # on demand, we init our pack cache - num_packs = len(pdb.entities()) - assert pdb._st_mtime != 0 - - # test pack directory changed: - # packs removed - rename a file, should affect the glob - pack_path = pdb.entities()[0].pack().path() - new_pack_path = pack_path + "renamed" - os.rename(pack_path, new_pack_path) - - pdb.update_cache(force=True) - assert len(pdb.entities()) == num_packs - 1 - - # packs added - os.rename(new_pack_path, pack_path) - pdb.update_cache(force=True) - assert len(pdb.entities()) == num_packs - - # bang on the cache - # access the Entities directly, as there is no iteration interface - # yet ( or required for now ) - sha_list = list(pdb.sha_iter()) - assert len(sha_list) == pdb.size() - - # hit all packs in random order - random.shuffle(sha_list) - - for sha in sha_list: - info = pdb.info(sha) - stream = pdb.stream(sha) - # END for each sha to query - - - # test short finding - be a bit more brutal here - max_bytes = 19 - min_bytes = 2 - num_ambiguous = 0 - for i, sha in enumerate(sha_list): - short_sha = sha[:max((i % max_bytes), min_bytes)] - try: - assert pdb.partial_to_complete_sha(short_sha, len(short_sha)*2) == sha - except AmbiguousObjectName: - num_ambiguous += 1 - pass # valid, we can have short objects - # END exception handling - # END for each sha to find - - # we should have at least one ambiguous, considering the small sizes - # but in our pack, there is no ambigious ... - # assert num_ambiguous - - # non-existing - self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) + + @with_rw_directory + @with_packs_rw + def test_writing(self, path): + pdb = PackedDB(path) + + # on demand, we init our pack cache + num_packs = len(pdb.entities()) + assert pdb._st_mtime != 0 + + # test pack directory changed: + # packs removed - rename a file, should affect the glob + pack_path = pdb.entities()[0].pack().path() + new_pack_path = pack_path + "renamed" + os.rename(pack_path, new_pack_path) + + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs - 1 + + # packs added + os.rename(new_pack_path, pack_path) + pdb.update_cache(force=True) + assert len(pdb.entities()) == num_packs + + # bang on the cache + # access the Entities directly, as there is no iteration interface + # yet ( or required for now ) + sha_list = list(pdb.sha_iter()) + assert len(sha_list) == pdb.size() + + # hit all packs in random order + random.shuffle(sha_list) + + for sha in sha_list: + info = pdb.info(sha) + stream = pdb.stream(sha) + # END for each sha to query + + + # test short finding - be a bit more brutal here + max_bytes = 19 + min_bytes = 2 + num_ambiguous = 0 + for i, sha in enumerate(sha_list): + short_sha = sha[:max((i % max_bytes), min_bytes)] + try: + assert pdb.partial_to_complete_sha(short_sha, len(short_sha)*2) == sha + except AmbiguousObjectName: + num_ambiguous += 1 + pass # valid, we can have short objects + # END exception handling + # END for each sha to find + + # we should have at least one ambiguous, considering the small sizes + # but in our pack, there is no ambigious ... + # assert num_ambiguous + + # non-existing + self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 0d8eeebb3..1637bff74 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -6,55 +6,55 @@ from gitdb.db import ReferenceDB from gitdb.util import ( - NULL_BIN_SHA, - hex_to_bin - ) + NULL_BIN_SHA, + hex_to_bin + ) import os - + class TestReferenceDB(TestDBBase): - - def make_alt_file(self, alt_path, alt_list): - """Create an alternates file which contains the given alternates. - The list can be empty""" - alt_file = open(alt_path, "wb") - for alt in alt_list: - alt_file.write(alt + "\n") - alt_file.close() - - @with_rw_directory - def test_writing(self, path): - NULL_BIN_SHA = '\0' * 20 - - alt_path = os.path.join(path, 'alternates') - rdb = ReferenceDB(alt_path) - assert len(rdb.databases()) == 0 - assert rdb.size() == 0 - assert len(list(rdb.sha_iter())) == 0 - - # try empty, non-existing - assert not rdb.has_object(NULL_BIN_SHA) - - - # setup alternate file - # add two, one is invalid - own_repo_path = fixture_path('../../../.git/objects') # use own repo - self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) - rdb.update_cache() - assert len(rdb.databases()) == 1 - - # we should now find a default revision of ours - gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") - assert rdb.has_object(gitdb_sha) - - # remove valid - self.make_alt_file(alt_path, ["just/one/invalid/path"]) - rdb.update_cache() - assert len(rdb.databases()) == 0 - - # add valid - self.make_alt_file(alt_path, [own_repo_path]) - rdb.update_cache() - assert len(rdb.databases()) == 1 - - + + def make_alt_file(self, alt_path, alt_list): + """Create an alternates file which contains the given alternates. + The list can be empty""" + alt_file = open(alt_path, "wb") + for alt in alt_list: + alt_file.write(alt + "\n") + alt_file.close() + + @with_rw_directory + def test_writing(self, path): + NULL_BIN_SHA = '\0' * 20 + + alt_path = os.path.join(path, 'alternates') + rdb = ReferenceDB(alt_path) + assert len(rdb.databases()) == 0 + assert rdb.size() == 0 + assert len(list(rdb.sha_iter())) == 0 + + # try empty, non-existing + assert not rdb.has_object(NULL_BIN_SHA) + + + # setup alternate file + # add two, one is invalid + own_repo_path = fixture_path('../../../.git/objects') # use own repo + self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 1 + + # we should now find a default revision of ours + gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") + assert rdb.has_object(gitdb_sha) + + # remove valid + self.make_alt_file(alt_path, ["just/one/invalid/path"]) + rdb.update_cache() + assert len(rdb.databases()) == 0 + + # add valid + self.make_alt_file(alt_path, [own_repo_path]) + rdb.update_cache() + assert len(rdb.databases()) == 1 + + diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 50645be65..ac8473a4e 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -4,12 +4,12 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" from gitdb import ( - OStream, - ) + OStream, + ) from gitdb.stream import ( - Sha1Writer, - ZippedStoreShaWriter - ) + Sha1Writer, + ZippedStoreShaWriter + ) from gitdb.util import zlib @@ -29,134 +29,134 @@ #{ Bases class TestBase(unittest.TestCase): - """Base class for all tests""" - + """Base class for all tests""" + #} END bases #{ Decorators def with_rw_directory(func): - """Create a temporary directory which can be written to, remove it if the - test suceeds, but leave it otherwise to aid additional debugging""" - def wrapper(self): - path = tempfile.mktemp(prefix=func.__name__) - os.mkdir(path) - keep = False - try: - try: - return func(self, path) - except Exception: - print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) - keep = True - raise - finally: - # Need to collect here to be sure all handles have been closed. It appears - # a windows-only issue. In fact things should be deleted, as well as - # memory maps closed, once objects go out of scope. For some reason - # though this is not the case here unless we collect explicitly. - if not keep: - gc.collect() - shutil.rmtree(path) - # END handle exception - # END wrapper - - wrapper.__name__ = func.__name__ - return wrapper + """Create a temporary directory which can be written to, remove it if the + test suceeds, but leave it otherwise to aid additional debugging""" + def wrapper(self): + path = tempfile.mktemp(prefix=func.__name__) + os.mkdir(path) + keep = False + try: + try: + return func(self, path) + except Exception: + print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + keep = True + raise + finally: + # Need to collect here to be sure all handles have been closed. It appears + # a windows-only issue. In fact things should be deleted, as well as + # memory maps closed, once objects go out of scope. For some reason + # though this is not the case here unless we collect explicitly. + if not keep: + gc.collect() + shutil.rmtree(path) + # END handle exception + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper def with_packs_rw(func): - """Function that provides a path into which the packs for testing should be - copied. Will pass on the path to the actual function afterwards""" - def wrapper(self, path): - src_pack_glob = fixture_path('packs/*') - copy_files_globbed(src_pack_glob, path, hard_link_ok=True) - return func(self, path) - # END wrapper - - wrapper.__name__ = func.__name__ - return wrapper + """Function that provides a path into which the packs for testing should be + copied. Will pass on the path to the actual function afterwards""" + def wrapper(self, path): + src_pack_glob = fixture_path('packs/*') + copy_files_globbed(src_pack_glob, path, hard_link_ok=True) + return func(self, path) + # END wrapper + + wrapper.__name__ = func.__name__ + return wrapper #} END decorators #{ Routines def fixture_path(relapath=''): - """:return: absolute path into the fixture directory - :param relapath: relative path into the fixtures directory, or '' - to obtain the fixture directory itself""" - return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) - + """:return: absolute path into the fixture directory + :param relapath: relative path into the fixtures directory, or '' + to obtain the fixture directory itself""" + return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) + def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): - """Copy all files found according to the given source glob into the target directory - :param hard_link_ok: if True, hard links will be created if possible. Otherwise - the files will be copied""" - for src_file in glob.glob(source_glob): - if hard_link_ok and hasattr(os, 'link'): - target = os.path.join(target_dir, os.path.basename(src_file)) - try: - os.link(src_file, target) - except OSError: - shutil.copy(src_file, target_dir) - # END handle cross device links ( and resulting failure ) - else: - shutil.copy(src_file, target_dir) - # END try hard link - # END for each file to copy - + """Copy all files found according to the given source glob into the target directory + :param hard_link_ok: if True, hard links will be created if possible. Otherwise + the files will be copied""" + for src_file in glob.glob(source_glob): + if hard_link_ok and hasattr(os, 'link'): + target = os.path.join(target_dir, os.path.basename(src_file)) + try: + os.link(src_file, target) + except OSError: + shutil.copy(src_file, target_dir) + # END handle cross device links ( and resulting failure ) + else: + shutil.copy(src_file, target_dir) + # END try hard link + # END for each file to copy + def make_bytes(size_in_bytes, randomize=False): - """:return: string with given size in bytes - :param randomize: try to produce a very random stream""" - actual_size = size_in_bytes / 4 - producer = xrange(actual_size) - if randomize: - producer = list(producer) - random.shuffle(producer) - # END randomize - a = array('i', producer) - return a.tostring() + """:return: string with given size in bytes + :param randomize: try to produce a very random stream""" + actual_size = size_in_bytes / 4 + producer = xrange(actual_size) + if randomize: + producer = list(producer) + random.shuffle(producer) + # END randomize + a = array('i', producer) + return a.tostring() def make_object(type, data): - """:return: bytes resembling an uncompressed object""" - odata = "blob %i\0" % len(data) - return odata + data - + """:return: bytes resembling an uncompressed object""" + odata = "blob %i\0" % len(data) + return odata + data + def make_memory_file(size_in_bytes, randomize=False): - """:return: tuple(size_of_stream, stream) - :param randomize: try to produce a very random stream""" - d = make_bytes(size_in_bytes, randomize) - return len(d), StringIO(d) + """:return: tuple(size_of_stream, stream) + :param randomize: try to produce a very random stream""" + d = make_bytes(size_in_bytes, randomize) + return len(d), StringIO(d) #} END routines #{ Stream Utilities class DummyStream(object): - def __init__(self): - self.was_read = False - self.bytes = 0 - self.closed = False - - def read(self, size): - self.was_read = True - self.bytes = size - - def close(self): - self.closed = True - - def _assert(self): - assert self.was_read + def __init__(self): + self.was_read = False + self.bytes = 0 + self.closed = False + + def read(self, size): + self.was_read = True + self.bytes = size + + def close(self): + self.closed = True + + def _assert(self): + assert self.was_read class DeriveTest(OStream): - def __init__(self, sha, type, size, stream, *args, **kwargs): - self.myarg = kwargs.pop('myarg') - self.args = args - - def _assert(self): - assert self.args - assert self.myarg + def __init__(self, sha, type, size, stream, *args, **kwargs): + self.myarg = kwargs.pop('myarg') + self.args = args + + def _assert(self): + assert self.args + assert self.myarg #} END stream utilitiess diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index 761113d51..3563fcfbe 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -16,12 +16,12 @@ #{ Utilities def resolve_or_fail(env_var): - """:return: resolved environment variable or raise EnvironmentError""" - try: - return os.environ[env_var] - except KeyError: - raise EnvironmentError("Please set the %r envrionment variable and retry" % env_var) - # END exception handling + """:return: resolved environment variable or raise EnvironmentError""" + try: + return os.environ[env_var] + except KeyError: + raise EnvironmentError("Please set the %r envrionment variable and retry" % env_var) + # END exception handling #} END utilities @@ -29,26 +29,26 @@ def resolve_or_fail(env_var): #{ Base Classes class TestBigRepoR(TestBase): - """TestCase providing access to readonly 'big' repositories using the following - member variables: - - * gitrepopath - - * read-only base path of the git source repository, i.e. .../git/.git""" - - #{ Invariants - head_sha_2k = '235d521da60e4699e5bd59ac658b5b48bd76ddca' - head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' - #} END invariants - - @classmethod - def setUpAll(cls): - try: - super(TestBigRepoR, cls).setUpAll() - except AttributeError: - pass - cls.gitrepopath = resolve_or_fail(k_env_git_repo) - assert cls.gitrepopath.endswith('.git') - - + """TestCase providing access to readonly 'big' repositories using the following + member variables: + + * gitrepopath + + * read-only base path of the git source repository, i.e. .../git/.git""" + + #{ Invariants + head_sha_2k = '235d521da60e4699e5bd59ac658b5b48bd76ddca' + head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' + #} END invariants + + @classmethod + def setUpAll(cls): + try: + super(TestBigRepoR, cls).setUpAll() + except AttributeError: + pass + cls.gitrepopath = resolve_or_fail(k_env_git_repo) + assert cls.gitrepopath.endswith('.git') + + #} END base classes diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index 20618024d..63856e218 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -4,8 +4,8 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance tests for object store""" from lib import ( - TestBigRepoR - ) + TestBigRepoR + ) from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB @@ -18,76 +18,76 @@ from nose import SkipTest class TestPackedDBPerformance(TestBigRepoR): - - def test_pack_random_access(self): - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - - # sha lookup - st = time() - sha_list = list(pdb.sha_iter()) - elapsed = time() - st - ns = len(sha_list) - print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) - - # sha lookup: best-case and worst case access - pdb_pack_info = pdb._pack_info - # END shuffle shas - st = time() - for sha in sha_list: - pdb_pack_info(sha) - # END for each sha to look up - elapsed = time() - st - - # discard cache - del(pdb._entities) - pdb.entities() - print >> sys.stderr, "PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed) - # END for each random mode - - # query info and streams only - max_items = 10000 # can wait longer when testing memory - for pdb_fun in (pdb.info, pdb.stream): - st = time() - for sha in sha_list[:max_items]: - pdb_fun(sha) - elapsed = time() - st - print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) - # END for each function - - # retrieve stream and read all - max_items = 5000 - pdb_stream = pdb.stream - total_size = 0 - st = time() - for sha in sha_list[:max_items]: - stream = pdb_stream(sha) - stream.read() - total_size += stream.size - elapsed = time() - st - total_kib = total_size / 1000 - print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) - - def test_correctness(self): - raise SkipTest("Takes too long, enable it if you change the algorithm and want to be sure you decode packs correctly") - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - # disabled for now as it used to work perfectly, checking big repositories takes a long time - print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" - for crc in range(2): - count = 0 - st = time() - for entity in pdb.entities(): - pack_verify = entity.is_valid_stream - sha_by_index = entity.index().sha - for index in xrange(entity.index().size()): - try: - assert pack_verify(sha_by_index(index), use_crc=crc) - count += 1 - except UnsupportedOperation: - pass - # END ignore old indices - # END for each index - # END for each entity - elapsed = time() - st - print >> sys.stderr, "PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed) - # END for each verify mode - + + def test_pack_random_access(self): + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + # sha lookup + st = time() + sha_list = list(pdb.sha_iter()) + elapsed = time() - st + ns = len(sha_list) + print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) + + # sha lookup: best-case and worst case access + pdb_pack_info = pdb._pack_info + # END shuffle shas + st = time() + for sha in sha_list: + pdb_pack_info(sha) + # END for each sha to look up + elapsed = time() - st + + # discard cache + del(pdb._entities) + pdb.entities() + print >> sys.stderr, "PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed) + # END for each random mode + + # query info and streams only + max_items = 10000 # can wait longer when testing memory + for pdb_fun in (pdb.info, pdb.stream): + st = time() + for sha in sha_list[:max_items]: + pdb_fun(sha) + elapsed = time() - st + print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) + # END for each function + + # retrieve stream and read all + max_items = 5000 + pdb_stream = pdb.stream + total_size = 0 + st = time() + for sha in sha_list[:max_items]: + stream = pdb_stream(sha) + stream.read() + total_size += stream.size + elapsed = time() - st + total_kib = total_size / 1000 + print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) + + def test_correctness(self): + raise SkipTest("Takes too long, enable it if you change the algorithm and want to be sure you decode packs correctly") + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + # disabled for now as it used to work perfectly, checking big repositories takes a long time + print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" + for crc in range(2): + count = 0 + st = time() + for entity in pdb.entities(): + pack_verify = entity.is_valid_stream + sha_by_index = entity.index().sha + for index in xrange(entity.index().size()): + try: + assert pack_verify(sha_by_index(index), use_crc=crc) + count += 1 + except UnsupportedOperation: + pass + # END ignore old indices + # END for each index + # END for each entity + elapsed = time() - st + print >> sys.stderr, "PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed) + # END for each verify mode + diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 3c40ed0fb..c66e60cba 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -4,8 +4,8 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Specific test for pack streams only""" from lib import ( - TestBigRepoR - ) + TestBigRepoR + ) from gitdb.db.pack import PackedDB from gitdb.stream import NullStream @@ -17,63 +17,63 @@ from nose import SkipTest class CountedNullStream(NullStream): - __slots__ = '_bw' - def __init__(self): - self._bw = 0 - - def bytes_written(self): - return self._bw - - def write(self, d): - self._bw += NullStream.write(self, d) - + __slots__ = '_bw' + def __init__(self): + self._bw = 0 + + def bytes_written(self): + return self._bw + + def write(self, d): + self._bw += NullStream.write(self, d) + class TestPackStreamingPerformance(TestBigRepoR): - - def test_pack_writing(self): - # see how fast we can write a pack from object streams. - # This will not be fast, as we take time for decompressing the streams as well - ostream = CountedNullStream() - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - - ni = 5000 - count = 0 - total_size = 0 - st = time() - for sha in pdb.sha_iter(): - count += 1 - pdb.stream(sha) - if count == ni: - break - #END gather objects for pack-writing - elapsed = time() - st - print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) - - st = time() - PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) - elapsed = time() - st - total_kb = ostream.bytes_written() / 1000 - print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) - - - def test_stream_reading(self): - raise SkipTest() - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - - # streaming only, meant for --with-profile runs - ni = 5000 - count = 0 - pdb_stream = pdb.stream - total_size = 0 - st = time() - for sha in pdb.sha_iter(): - if count == ni: - break - stream = pdb_stream(sha) - stream.read() - total_size += stream.size - count += 1 - elapsed = time() - st - total_kib = total_size / 1000 - print >> sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed) - + + def test_pack_writing(self): + # see how fast we can write a pack from object streams. + # This will not be fast, as we take time for decompressing the streams as well + ostream = CountedNullStream() + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + ni = 5000 + count = 0 + total_size = 0 + st = time() + for sha in pdb.sha_iter(): + count += 1 + pdb.stream(sha) + if count == ni: + break + #END gather objects for pack-writing + elapsed = time() - st + print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) + + st = time() + PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) + elapsed = time() - st + total_kb = ostream.bytes_written() / 1000 + print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) + + + def test_stream_reading(self): + raise SkipTest() + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + + # streaming only, meant for --with-profile runs + ni = 5000 + count = 0 + pdb_stream = pdb.stream + total_size = 0 + st = time() + for sha in pdb.sha_iter(): + if count == ni: + break + stream = pdb_stream(sha) + stream.read() + total_size += stream.size + count += 1 + elapsed = time() - st + total_kib = total_size / 1000 + print >> sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed) + diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index f5f2e2e4d..010003d4b 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -8,16 +8,16 @@ from gitdb.base import * from gitdb.stream import * from gitdb.util import ( - pool, - bin_to_hex - ) + pool, + bin_to_hex + ) from gitdb.typ import str_blob_type from gitdb.fun import chunk_size from async import ( - IteratorReader, - ChannelThreadTask, - ) + IteratorReader, + ChannelThreadTask, + ) from cStringIO import StringIO from time import time @@ -28,168 +28,168 @@ from lib import ( - TestBigRepoR, - make_memory_file, - with_rw_directory - ) + TestBigRepoR, + make_memory_file, + with_rw_directory + ) #{ Utilities def read_chunked_stream(stream): - total = 0 - while True: - chunk = stream.read(chunk_size) - total += len(chunk) - if len(chunk) < chunk_size: - break - # END read stream loop - assert total == stream.size - return stream - - + total = 0 + while True: + chunk = stream.read(chunk_size) + total += len(chunk) + if len(chunk) < chunk_size: + break + # END read stream loop + assert total == stream.size + return stream + + class TestStreamReader(ChannelThreadTask): - """Expects input streams and reads them in chunks. It will read one at a time, - requireing a queue chunk of size 1""" - def __init__(self, *args): - super(TestStreamReader, self).__init__(*args) - self.fun = read_chunked_stream - self.max_chunksize = 1 - + """Expects input streams and reads them in chunks. It will read one at a time, + requireing a queue chunk of size 1""" + def __init__(self, *args): + super(TestStreamReader, self).__init__(*args) + self.fun = read_chunked_stream + self.max_chunksize = 1 + #} END utilities class TestObjDBPerformance(TestBigRepoR): - - large_data_size_bytes = 1000*1000*50 # some MiB should do it - moderate_data_size_bytes = 1000*1000*1 # just 1 MiB - - @with_rw_directory - def test_large_data_streaming(self, path): - ldb = LooseObjectDB(path) - string_ios = list() # list of streams we previously created - - # serial mode - for randomize in range(2): - desc = (randomize and 'random ') or '' - print >> sys.stderr, "Creating %s data ..." % desc - st = time() - size, stream = make_memory_file(self.large_data_size_bytes, randomize) - elapsed = time() - st - print >> sys.stderr, "Done (in %f s)" % elapsed - string_ios.append(stream) - - # writing - due to the compression it will seem faster than it is - st = time() - sha = ldb.store(IStream('blob', size, stream)).binsha - elapsed_add = time() - st - assert ldb.has_object(sha) - db_file = ldb.readable_db_object_path(bin_to_hex(sha)) - fsize_kib = os.path.getsize(db_file) / 1000 - - - size_kib = size / 1000 - print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) - - # reading all at once - st = time() - ostream = ldb.stream(sha) - shadata = ostream.read() - elapsed_readall = time() - st - - stream.seek(0) - assert shadata == stream.getvalue() - print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) - - - # reading in chunks of 1 MiB - cs = 512*1000 - chunks = list() - st = time() - ostream = ldb.stream(sha) - while True: - data = ostream.read(cs) - chunks.append(data) - if len(data) < cs: - break - # END read in chunks - elapsed_readchunks = time() - st - - stream.seek(0) - assert ''.join(chunks) == stream.getvalue() - - cs_kib = cs / 1000 - print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) - - # del db file so we keep something to do - os.remove(db_file) - # END for each randomization factor - - - # multi-threaded mode - # want two, should be supported by most of todays cpus - pool.set_size(2) - total_kib = 0 - nsios = len(string_ios) - for stream in string_ios: - stream.seek(0) - total_kib += len(stream.getvalue()) / 1000 - # END rewind - - def istream_iter(): - for stream in string_ios: - stream.seek(0) - yield IStream(str_blob_type, len(stream.getvalue()), stream) - # END for each stream - # END util - - # write multiple objects at once, involving concurrent compression - reader = IteratorReader(istream_iter()) - istream_reader = ldb.store_async(reader) - istream_reader.task().max_chunksize = 1 - - st = time() - istreams = istream_reader.read(nsios) - assert len(istreams) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - - # decompress multiple at once, by reading them - # chunk size is not important as the stream will not really be decompressed - - # until its read - istream_reader = IteratorReader(iter([ i.binsha for i in istreams ])) - ostream_reader = ldb.stream_async(istream_reader) - - chunk_task = TestStreamReader(ostream_reader, "chunker", None) - output_reader = pool.add_task(chunk_task) - output_reader.task().max_chunksize = 1 - - st = time() - assert len(output_reader.read(nsios)) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Read KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - - # store the files, and read them back. For the reading, we use a task - # as well which is chunked into one item per task. Reading all will - # very quickly result in two threads handling two bytestreams of - # chained compression/decompression streams - reader = IteratorReader(istream_iter()) - istream_reader = ldb.store_async(reader) - istream_reader.task().max_chunksize = 1 - - istream_to_sha = lambda items: [ i.binsha for i in items ] - istream_reader.set_post_cb(istream_to_sha) - - ostream_reader = ldb.stream_async(istream_reader) - - chunk_task = TestStreamReader(ostream_reader, "chunker", None) - output_reader = pool.add_task(chunk_task) - output_reader.max_chunksize = 1 - - st = time() - assert len(output_reader.read(nsios)) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Compressed and decompressed and read %i KiB of data in loose odb in %f s ( %f Combined KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + large_data_size_bytes = 1000*1000*50 # some MiB should do it + moderate_data_size_bytes = 1000*1000*1 # just 1 MiB + + @with_rw_directory + def test_large_data_streaming(self, path): + ldb = LooseObjectDB(path) + string_ios = list() # list of streams we previously created + + # serial mode + for randomize in range(2): + desc = (randomize and 'random ') or '' + print >> sys.stderr, "Creating %s data ..." % desc + st = time() + size, stream = make_memory_file(self.large_data_size_bytes, randomize) + elapsed = time() - st + print >> sys.stderr, "Done (in %f s)" % elapsed + string_ios.append(stream) + + # writing - due to the compression it will seem faster than it is + st = time() + sha = ldb.store(IStream('blob', size, stream)).binsha + elapsed_add = time() - st + assert ldb.has_object(sha) + db_file = ldb.readable_db_object_path(bin_to_hex(sha)) + fsize_kib = os.path.getsize(db_file) / 1000 + + + size_kib = size / 1000 + print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) + + # reading all at once + st = time() + ostream = ldb.stream(sha) + shadata = ostream.read() + elapsed_readall = time() - st + + stream.seek(0) + assert shadata == stream.getvalue() + print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) + + + # reading in chunks of 1 MiB + cs = 512*1000 + chunks = list() + st = time() + ostream = ldb.stream(sha) + while True: + data = ostream.read(cs) + chunks.append(data) + if len(data) < cs: + break + # END read in chunks + elapsed_readchunks = time() - st + + stream.seek(0) + assert ''.join(chunks) == stream.getvalue() + + cs_kib = cs / 1000 + print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) + + # del db file so we keep something to do + os.remove(db_file) + # END for each randomization factor + + + # multi-threaded mode + # want two, should be supported by most of todays cpus + pool.set_size(2) + total_kib = 0 + nsios = len(string_ios) + for stream in string_ios: + stream.seek(0) + total_kib += len(stream.getvalue()) / 1000 + # END rewind + + def istream_iter(): + for stream in string_ios: + stream.seek(0) + yield IStream(str_blob_type, len(stream.getvalue()), stream) + # END for each stream + # END util + + # write multiple objects at once, involving concurrent compression + reader = IteratorReader(istream_iter()) + istream_reader = ldb.store_async(reader) + istream_reader.task().max_chunksize = 1 + + st = time() + istreams = istream_reader.read(nsios) + assert len(istreams) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + # decompress multiple at once, by reading them + # chunk size is not important as the stream will not really be decompressed + + # until its read + istream_reader = IteratorReader(iter([ i.binsha for i in istreams ])) + ostream_reader = ldb.stream_async(istream_reader) + + chunk_task = TestStreamReader(ostream_reader, "chunker", None) + output_reader = pool.add_task(chunk_task) + output_reader.task().max_chunksize = 1 + + st = time() + assert len(output_reader.read(nsios)) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Read KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) + + # store the files, and read them back. For the reading, we use a task + # as well which is chunked into one item per task. Reading all will + # very quickly result in two threads handling two bytestreams of + # chained compression/decompression streams + reader = IteratorReader(istream_iter()) + istream_reader = ldb.store_async(reader) + istream_reader.task().max_chunksize = 1 + + istream_to_sha = lambda items: [ i.binsha for i in items ] + istream_reader.set_post_cb(istream_to_sha) + + ostream_reader = ldb.stream_async(istream_reader) + + chunk_task = TestStreamReader(ostream_reader, "chunker", None) + output_reader = pool.add_task(chunk_task) + output_reader.max_chunksize = 1 + + st = time() + assert len(output_reader.read(nsios)) == nsios + elapsed = time() - st + + print >> sys.stderr, "Threads(%i): Compressed and decompressed and read %i KiB of data in loose odb in %f s ( %f Combined KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 1b20faf87..d4ce428c3 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -4,95 +4,95 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from lib import ( - TestBase, - DummyStream, - DeriveTest, - ) + TestBase, + DummyStream, + DeriveTest, + ) from gitdb import * from gitdb.util import ( - NULL_BIN_SHA - ) + NULL_BIN_SHA + ) from gitdb.typ import ( - str_blob_type - ) + str_blob_type + ) class TestBaseTypes(TestBase): - - def test_streams(self): - # test info - sha = NULL_BIN_SHA - s = 20 - blob_id = 3 - - info = OInfo(sha, str_blob_type, s) - assert info.binsha == sha - assert info.type == str_blob_type - assert info.type_id == blob_id - assert info.size == s - - # test pack info - # provides type_id - pinfo = OPackInfo(0, blob_id, s) - assert pinfo.type == str_blob_type - assert pinfo.type_id == blob_id - assert pinfo.pack_offset == 0 - - dpinfo = ODeltaPackInfo(0, blob_id, s, sha) - assert dpinfo.type == str_blob_type - assert dpinfo.type_id == blob_id - assert dpinfo.delta_info == sha - assert dpinfo.pack_offset == 0 - - - # test ostream - stream = DummyStream() - ostream = OStream(*(info + (stream, ))) - assert ostream.stream is stream - ostream.read(15) - stream._assert() - assert stream.bytes == 15 - ostream.read(20) - assert stream.bytes == 20 - - # test packstream - postream = OPackStream(*(pinfo + (stream, ))) - assert postream.stream is stream - postream.read(10) - stream._assert() - assert stream.bytes == 10 - - # test deltapackstream - dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) - dpostream.stream is stream - dpostream.read(5) - stream._assert() - assert stream.bytes == 5 - - # derive with own args - DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() - - # test istream - istream = IStream(str_blob_type, s, stream) - assert istream.binsha == None - istream.binsha = sha - assert istream.binsha == sha - - assert len(istream.binsha) == 20 - assert len(istream.hexsha) == 40 - - assert istream.size == s - istream.size = s * 2 - istream.size == s * 2 - assert istream.type == str_blob_type - istream.type = "something" - assert istream.type == "something" - assert istream.stream is stream - istream.stream = None - assert istream.stream is None - - assert istream.error is None - istream.error = Exception() - assert isinstance(istream.error, Exception) + + def test_streams(self): + # test info + sha = NULL_BIN_SHA + s = 20 + blob_id = 3 + + info = OInfo(sha, str_blob_type, s) + assert info.binsha == sha + assert info.type == str_blob_type + assert info.type_id == blob_id + assert info.size == s + + # test pack info + # provides type_id + pinfo = OPackInfo(0, blob_id, s) + assert pinfo.type == str_blob_type + assert pinfo.type_id == blob_id + assert pinfo.pack_offset == 0 + + dpinfo = ODeltaPackInfo(0, blob_id, s, sha) + assert dpinfo.type == str_blob_type + assert dpinfo.type_id == blob_id + assert dpinfo.delta_info == sha + assert dpinfo.pack_offset == 0 + + + # test ostream + stream = DummyStream() + ostream = OStream(*(info + (stream, ))) + assert ostream.stream is stream + ostream.read(15) + stream._assert() + assert stream.bytes == 15 + ostream.read(20) + assert stream.bytes == 20 + + # test packstream + postream = OPackStream(*(pinfo + (stream, ))) + assert postream.stream is stream + postream.read(10) + stream._assert() + assert stream.bytes == 10 + + # test deltapackstream + dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) + dpostream.stream is stream + dpostream.read(5) + stream._assert() + assert stream.bytes == 5 + + # derive with own args + DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() + + # test istream + istream = IStream(str_blob_type, s, stream) + assert istream.binsha == None + istream.binsha = sha + assert istream.binsha == sha + + assert len(istream.binsha) == 20 + assert len(istream.hexsha) == 40 + + assert istream.size == s + istream.size = s * 2 + istream.size == s * 2 + assert istream.type == str_blob_type + istream.type = "something" + assert istream.type == "something" + assert istream.stream is stream + istream.stream = None + assert istream.stream is None + + assert istream.error is None + istream.error = Exception() + assert isinstance(istream.error, Exception) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 753177560..611ae4299 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -7,58 +7,58 @@ from gitdb import IStream from gitdb.db import LooseObjectDB from gitdb.util import pool - + from cStringIO import StringIO from async import IteratorReader - + class TestExamples(TestBase): - - def test_base(self): - ldb = LooseObjectDB(fixture_path("../../../.git/objects")) - - for sha1 in ldb.sha_iter(): - oinfo = ldb.info(sha1) - ostream = ldb.stream(sha1) - assert oinfo[:3] == ostream[:3] - - assert len(ostream.read()) == ostream.size - assert ldb.has_object(oinfo.binsha) - # END for each sha in database - # assure we close all files - try: - del(ostream) - del(oinfo) - except UnboundLocalError: - pass - # END ignore exception if there are no loose objects - - data = "my data" - istream = IStream("blob", len(data), StringIO(data)) - - # the object does not yet have a sha - assert istream.binsha is None - ldb.store(istream) - # now the sha is set - assert len(istream.binsha) == 20 - assert ldb.has_object(istream.binsha) - - - # async operation - # Create a reader from an iterator - reader = IteratorReader(ldb.sha_iter()) - - # get reader for object streams - info_reader = ldb.stream_async(reader) - - # read one - info = info_reader.read(1)[0] - - # read all the rest until depletion - ostreams = info_reader.read() - - # set the pool to use two threads - pool.set_size(2) - - # synchronize the mode of operation - pool.set_size(0) + + def test_base(self): + ldb = LooseObjectDB(fixture_path("../../../.git/objects")) + + for sha1 in ldb.sha_iter(): + oinfo = ldb.info(sha1) + ostream = ldb.stream(sha1) + assert oinfo[:3] == ostream[:3] + + assert len(ostream.read()) == ostream.size + assert ldb.has_object(oinfo.binsha) + # END for each sha in database + # assure we close all files + try: + del(ostream) + del(oinfo) + except UnboundLocalError: + pass + # END ignore exception if there are no loose objects + + data = "my data" + istream = IStream("blob", len(data), StringIO(data)) + + # the object does not yet have a sha + assert istream.binsha is None + ldb.store(istream) + # now the sha is set + assert len(istream.binsha) == 20 + assert ldb.has_object(istream.binsha) + + + # async operation + # Create a reader from an iterator + reader = IteratorReader(ldb.sha_iter()) + + # get reader for object streams + info_reader = ldb.stream_async(reader) + + # read one + info = info_reader.read(1)[0] + + # read all the rest until depletion + ostreams = info_reader.read() + + # set the pool to use two threads + pool.set_size(2) + + # synchronize the mode of operation + pool.set_size(0) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 4a7f1caf2..779155a2a 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -4,23 +4,23 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test everything about packs reading and writing""" from lib import ( - TestBase, - with_rw_directory, - with_packs_rw, - fixture_path - ) + TestBase, + with_rw_directory, + with_packs_rw, + fixture_path + ) from gitdb.stream import DeltaApplyReader from gitdb.pack import ( - PackEntity, - PackIndexFile, - PackFile - ) + PackEntity, + PackIndexFile, + PackFile + ) from gitdb.base import ( - OInfo, - OStream, - ) + OInfo, + OStream, + ) from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation @@ -35,213 +35,213 @@ #{ Utilities def bin_sha_from_filename(filename): - return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:]) + return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:]) #} END utilities class TestPack(TestBase): - - packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) - packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) - packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) - packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) - packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) - packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) - - - def _assert_index_file(self, index, version, size): - assert index.packfile_checksum() != index.indexfile_checksum() - assert len(index.packfile_checksum()) == 20 - assert len(index.indexfile_checksum()) == 20 - assert index.version() == version - assert index.size() == size - assert len(index.offsets()) == size - - # get all data of all objects - for oidx in xrange(index.size()): - sha = index.sha(oidx) - assert oidx == index.sha_to_index(sha) - - entry = index.entry(oidx) - assert len(entry) == 3 - - assert entry[0] == index.offset(oidx) - assert entry[1] == sha - assert entry[2] == index.crc(oidx) - - # verify partial sha - for l in (4,8,11,17,20): - assert index.partial_sha_to_index(sha[:l], l*2) == oidx - - # END for each object index in indexfile - self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) - - - def _assert_pack_file(self, pack, version, size): - assert pack.version() == 2 - assert pack.size() == size - assert len(pack.checksum()) == 20 - - num_obj = 0 - for obj in pack.stream_iter(): - num_obj += 1 - info = pack.info(obj.pack_offset) - stream = pack.stream(obj.pack_offset) - - assert info.pack_offset == stream.pack_offset - assert info.type_id == stream.type_id - assert hasattr(stream, 'read') - - # it should be possible to read from both streams - assert obj.read() == stream.read() - - streams = pack.collect_streams(obj.pack_offset) - assert streams - - # read the stream - try: - dstream = DeltaApplyReader.new(streams) - except ValueError: - # ignore these, old git versions use only ref deltas, - # which we havent resolved ( as we are without an index ) - # Also ignore non-delta streams - continue - # END get deltastream - - # read all - data = dstream.read() - assert len(data) == dstream.size - - # test seek - dstream.seek(0) - assert dstream.read() == data - - - # read chunks - # NOTE: the current implementation is safe, it basically transfers - # all calls to the underlying memory map - - # END for each object - assert num_obj == size - - - def test_pack_index(self): - # check version 1 and 2 - for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): - index = PackIndexFile(indexfile) - self._assert_index_file(index, version, size) - # END run tests - - def test_pack(self): - # there is this special version 3, but apparently its like 2 ... - for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): - pack = PackFile(packfile) - self._assert_pack_file(pack, version, size) - # END for each pack to test - - @with_rw_directory - def test_pack_entity(self, rw_dir): - pack_objs = list() - for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), - (self.packfile_v2_2, self.packindexfile_v2), - (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): - packfile, version, size = packinfo - indexfile, version, size = indexinfo - entity = PackEntity(packfile) - assert entity.pack().path() == packfile - assert entity.index().path() == indexfile - pack_objs.extend(entity.stream_iter()) - - count = 0 - for info, stream in izip(entity.info_iter(), entity.stream_iter()): - count += 1 - assert info.binsha == stream.binsha - assert len(info.binsha) == 20 - assert info.type_id == stream.type_id - assert info.size == stream.size - - # we return fully resolved items, which is implied by the sha centric access - assert not info.type_id in delta_types - - # try all calls - assert len(entity.collect_streams(info.binsha)) - oinfo = entity.info(info.binsha) - assert isinstance(oinfo, OInfo) - assert oinfo.binsha is not None - ostream = entity.stream(info.binsha) - assert isinstance(ostream, OStream) - assert ostream.binsha is not None - - # verify the stream - try: - assert entity.is_valid_stream(info.binsha, use_crc=True) - except UnsupportedOperation: - pass - # END ignore version issues - assert entity.is_valid_stream(info.binsha, use_crc=False) - # END for each info, stream tuple - assert count == size - - # END for each entity - - # pack writing - write all packs into one - # index path can be None - pack_path = tempfile.mktemp('', "pack", rw_dir) - index_path = tempfile.mktemp('', 'index', rw_dir) - iteration = 0 - def rewind_streams(): - for obj in pack_objs: - obj.stream.seek(0) - #END utility - for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): - pfile = open(ppath, 'wb') - iwrite = None - if ipath: - ifile = open(ipath, 'wb') - iwrite = ifile.write - #END handle ip - - # make sure we rewind the streams ... we work on the same objects over and over again - if iteration > 0: - rewind_streams() - #END rewind streams - iteration += 1 - - pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) - pfile.close() - assert os.path.getsize(ppath) > 100 - - # verify pack - pf = PackFile(ppath) - assert pf.size() == len(pack_objs) - assert pf.version() == PackFile.pack_version_default - assert pf.checksum() == pack_sha - - # verify index - if ipath is not None: - ifile.close() - assert os.path.getsize(ipath) > 100 - idx = PackIndexFile(ipath) - assert idx.version() == PackIndexFile.index_version_default - assert idx.packfile_checksum() == pack_sha - assert idx.indexfile_checksum() == index_sha - assert idx.size() == len(pack_objs) - #END verify files exist - #END for each packpath, indexpath pair - - # verify the packs throughly - rewind_streams() - entity = PackEntity.create(pack_objs, rw_dir) - count = 0 - for info in entity.info_iter(): - count += 1 - for use_crc in range(2): - assert entity.is_valid_stream(info.binsha, use_crc) - # END for each crc mode - #END for each info - assert count == len(pack_objs) - - - def test_pack_64(self): - # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets - # of course without really needing such a huge pack - raise SkipTest() + + packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) + packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) + packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) + packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) + packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) + packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) + + + def _assert_index_file(self, index, version, size): + assert index.packfile_checksum() != index.indexfile_checksum() + assert len(index.packfile_checksum()) == 20 + assert len(index.indexfile_checksum()) == 20 + assert index.version() == version + assert index.size() == size + assert len(index.offsets()) == size + + # get all data of all objects + for oidx in xrange(index.size()): + sha = index.sha(oidx) + assert oidx == index.sha_to_index(sha) + + entry = index.entry(oidx) + assert len(entry) == 3 + + assert entry[0] == index.offset(oidx) + assert entry[1] == sha + assert entry[2] == index.crc(oidx) + + # verify partial sha + for l in (4,8,11,17,20): + assert index.partial_sha_to_index(sha[:l], l*2) == oidx + + # END for each object index in indexfile + self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) + + + def _assert_pack_file(self, pack, version, size): + assert pack.version() == 2 + assert pack.size() == size + assert len(pack.checksum()) == 20 + + num_obj = 0 + for obj in pack.stream_iter(): + num_obj += 1 + info = pack.info(obj.pack_offset) + stream = pack.stream(obj.pack_offset) + + assert info.pack_offset == stream.pack_offset + assert info.type_id == stream.type_id + assert hasattr(stream, 'read') + + # it should be possible to read from both streams + assert obj.read() == stream.read() + + streams = pack.collect_streams(obj.pack_offset) + assert streams + + # read the stream + try: + dstream = DeltaApplyReader.new(streams) + except ValueError: + # ignore these, old git versions use only ref deltas, + # which we havent resolved ( as we are without an index ) + # Also ignore non-delta streams + continue + # END get deltastream + + # read all + data = dstream.read() + assert len(data) == dstream.size + + # test seek + dstream.seek(0) + assert dstream.read() == data + + + # read chunks + # NOTE: the current implementation is safe, it basically transfers + # all calls to the underlying memory map + + # END for each object + assert num_obj == size + + + def test_pack_index(self): + # check version 1 and 2 + for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): + index = PackIndexFile(indexfile) + self._assert_index_file(index, version, size) + # END run tests + + def test_pack(self): + # there is this special version 3, but apparently its like 2 ... + for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): + pack = PackFile(packfile) + self._assert_pack_file(pack, version, size) + # END for each pack to test + + @with_rw_directory + def test_pack_entity(self, rw_dir): + pack_objs = list() + for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), + (self.packfile_v2_2, self.packindexfile_v2), + (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): + packfile, version, size = packinfo + indexfile, version, size = indexinfo + entity = PackEntity(packfile) + assert entity.pack().path() == packfile + assert entity.index().path() == indexfile + pack_objs.extend(entity.stream_iter()) + + count = 0 + for info, stream in izip(entity.info_iter(), entity.stream_iter()): + count += 1 + assert info.binsha == stream.binsha + assert len(info.binsha) == 20 + assert info.type_id == stream.type_id + assert info.size == stream.size + + # we return fully resolved items, which is implied by the sha centric access + assert not info.type_id in delta_types + + # try all calls + assert len(entity.collect_streams(info.binsha)) + oinfo = entity.info(info.binsha) + assert isinstance(oinfo, OInfo) + assert oinfo.binsha is not None + ostream = entity.stream(info.binsha) + assert isinstance(ostream, OStream) + assert ostream.binsha is not None + + # verify the stream + try: + assert entity.is_valid_stream(info.binsha, use_crc=True) + except UnsupportedOperation: + pass + # END ignore version issues + assert entity.is_valid_stream(info.binsha, use_crc=False) + # END for each info, stream tuple + assert count == size + + # END for each entity + + # pack writing - write all packs into one + # index path can be None + pack_path = tempfile.mktemp('', "pack", rw_dir) + index_path = tempfile.mktemp('', 'index', rw_dir) + iteration = 0 + def rewind_streams(): + for obj in pack_objs: + obj.stream.seek(0) + #END utility + for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): + pfile = open(ppath, 'wb') + iwrite = None + if ipath: + ifile = open(ipath, 'wb') + iwrite = ifile.write + #END handle ip + + # make sure we rewind the streams ... we work on the same objects over and over again + if iteration > 0: + rewind_streams() + #END rewind streams + iteration += 1 + + pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) + pfile.close() + assert os.path.getsize(ppath) > 100 + + # verify pack + pf = PackFile(ppath) + assert pf.size() == len(pack_objs) + assert pf.version() == PackFile.pack_version_default + assert pf.checksum() == pack_sha + + # verify index + if ipath is not None: + ifile.close() + assert os.path.getsize(ipath) > 100 + idx = PackIndexFile(ipath) + assert idx.version() == PackIndexFile.index_version_default + assert idx.packfile_checksum() == pack_sha + assert idx.indexfile_checksum() == index_sha + assert idx.size() == len(pack_objs) + #END verify files exist + #END for each packpath, indexpath pair + + # verify the packs throughly + rewind_streams() + entity = PackEntity.create(pack_objs, rw_dir) + count = 0 + for info in entity.info_iter(): + count += 1 + for use_crc in range(2): + assert entity.is_valid_stream(info.binsha, use_crc) + # END for each crc mode + #END for each info + assert count == len(pack_objs) + + + def test_pack_64(self): + # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets + # of course without really needing such a huge pack + raise SkipTest() diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 523f77056..6dc27463c 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -4,24 +4,24 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from lib import ( - TestBase, - DummyStream, - Sha1Writer, - make_bytes, - make_object, - fixture_path - ) + TestBase, + DummyStream, + Sha1Writer, + make_bytes, + make_object, + fixture_path + ) from gitdb import * from gitdb.util import ( - NULL_HEX_SHA, - hex_to_bin - ) + NULL_HEX_SHA, + hex_to_bin + ) from gitdb.util import zlib from gitdb.typ import ( - str_blob_type - ) + str_blob_type + ) import time import tempfile @@ -31,124 +31,124 @@ class TestStream(TestBase): - """Test stream classes""" - - data_sizes = (15, 10000, 1000*1024+512) - - def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): - """Make stream tests - the orig_stream is seekable, allowing it to be - rewound and reused - :param cdata: the data we expect to read from stream, the contents - :param rewind_stream: function called to rewind the stream to make it ready - for reuse""" - ns = 10 - assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) - - # read in small steps - ss = len(cdata) / ns - for i in range(ns): - data = stream.read(ss) - chunk = cdata[i*ss:(i+1)*ss] - assert data == chunk - # END for each step - rest = stream.read() - if rest: - assert rest == cdata[-len(rest):] - # END handle rest - - if isinstance(stream, DecompressMemMapReader): - assert len(stream.data()) == stream.compressed_bytes_read() - # END handle special type - - rewind_stream(stream) - - # read everything - rdata = stream.read() - assert rdata == cdata - - if isinstance(stream, DecompressMemMapReader): - assert len(stream.data()) == stream.compressed_bytes_read() - # END handle special type - - def test_decompress_reader(self): - for close_on_deletion in range(2): - for with_size in range(2): - for ds in self.data_sizes: - cdata = make_bytes(ds, randomize=False) - - # zdata = zipped actual data - # cdata = original content data - - # create reader - if with_size: - # need object data - zdata = zlib.compress(make_object(str_blob_type, cdata)) - type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) - assert size == len(cdata) - assert type == str_blob_type - - # even if we don't set the size, it will be set automatically on first read - test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) - assert test_reader._s == len(cdata) - else: - # here we need content data - zdata = zlib.compress(cdata) - reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) - assert reader._s == len(cdata) - # END get reader - - self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) - - # put in a dummy stream for closing - dummy = DummyStream() - reader._m = dummy - - assert not dummy.closed - del(reader) - assert dummy.closed == close_on_deletion - # END for each datasize - # END whether size should be used - # END whether stream should be closed when deleted - - def test_sha_writer(self): - writer = Sha1Writer() - assert 2 == writer.write("hi") - assert len(writer.sha(as_hex=1)) == 40 - assert len(writer.sha(as_hex=0)) == 20 - - # make sure it does something ;) - prev_sha = writer.sha() - writer.write("hi again") - assert writer.sha() != prev_sha - - def test_compressed_writer(self): - for ds in self.data_sizes: - fd, path = tempfile.mkstemp() - ostream = FDCompressedSha1Writer(fd) - data = make_bytes(ds, randomize=False) - - # for now, just a single write, code doesn't care about chunking - assert len(data) == ostream.write(data) - ostream.close() - - # its closed already - self.failUnlessRaises(OSError, os.close, fd) - - # read everything back, compare to data we zip - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) - written_data = os.read(fd, os.path.getsize(path)) - assert len(written_data) == os.path.getsize(path) - os.close(fd) - assert written_data == zlib.compress(data, 1) # best speed - - os.remove(path) - # END for each os - - def test_decompress_reader_special_case(self): - odb = LooseObjectDB(fixture_path('objects')) - ostream = odb.stream(hex_to_bin('7bb839852ed5e3a069966281bb08d50012fb309b')) - - # if there is a bug, we will be missing one byte exactly ! - data = ostream.read() - assert len(data) == ostream.size - + """Test stream classes""" + + data_sizes = (15, 10000, 1000*1024+512) + + def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): + """Make stream tests - the orig_stream is seekable, allowing it to be + rewound and reused + :param cdata: the data we expect to read from stream, the contents + :param rewind_stream: function called to rewind the stream to make it ready + for reuse""" + ns = 10 + assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) + + # read in small steps + ss = len(cdata) / ns + for i in range(ns): + data = stream.read(ss) + chunk = cdata[i*ss:(i+1)*ss] + assert data == chunk + # END for each step + rest = stream.read() + if rest: + assert rest == cdata[-len(rest):] + # END handle rest + + if isinstance(stream, DecompressMemMapReader): + assert len(stream.data()) == stream.compressed_bytes_read() + # END handle special type + + rewind_stream(stream) + + # read everything + rdata = stream.read() + assert rdata == cdata + + if isinstance(stream, DecompressMemMapReader): + assert len(stream.data()) == stream.compressed_bytes_read() + # END handle special type + + def test_decompress_reader(self): + for close_on_deletion in range(2): + for with_size in range(2): + for ds in self.data_sizes: + cdata = make_bytes(ds, randomize=False) + + # zdata = zipped actual data + # cdata = original content data + + # create reader + if with_size: + # need object data + zdata = zlib.compress(make_object(str_blob_type, cdata)) + type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) + assert size == len(cdata) + assert type == str_blob_type + + # even if we don't set the size, it will be set automatically on first read + test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) + assert test_reader._s == len(cdata) + else: + # here we need content data + zdata = zlib.compress(cdata) + reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) + assert reader._s == len(cdata) + # END get reader + + self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) + + # put in a dummy stream for closing + dummy = DummyStream() + reader._m = dummy + + assert not dummy.closed + del(reader) + assert dummy.closed == close_on_deletion + # END for each datasize + # END whether size should be used + # END whether stream should be closed when deleted + + def test_sha_writer(self): + writer = Sha1Writer() + assert 2 == writer.write("hi") + assert len(writer.sha(as_hex=1)) == 40 + assert len(writer.sha(as_hex=0)) == 20 + + # make sure it does something ;) + prev_sha = writer.sha() + writer.write("hi again") + assert writer.sha() != prev_sha + + def test_compressed_writer(self): + for ds in self.data_sizes: + fd, path = tempfile.mkstemp() + ostream = FDCompressedSha1Writer(fd) + data = make_bytes(ds, randomize=False) + + # for now, just a single write, code doesn't care about chunking + assert len(data) == ostream.write(data) + ostream.close() + + # its closed already + self.failUnlessRaises(OSError, os.close, fd) + + # read everything back, compare to data we zip + fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + written_data = os.read(fd, os.path.getsize(path)) + assert len(written_data) == os.path.getsize(path) + os.close(fd) + assert written_data == zlib.compress(data, 1) # best speed + + os.remove(path) + # END for each os + + def test_decompress_reader_special_case(self): + odb = LooseObjectDB(fixture_path('objects')) + ostream = odb.stream(hex_to_bin('7bb839852ed5e3a069966281bb08d50012fb309b')) + + # if there is a bug, we will be missing one byte exactly ! + data = ostream.read() + assert len(data) == ostream.size + diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 90f4156b9..35f9f44a7 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -8,98 +8,98 @@ from lib import TestBase from gitdb.util import ( - to_hex_sha, - to_bin_sha, - NULL_HEX_SHA, - LockedFD - ) + to_hex_sha, + to_bin_sha, + NULL_HEX_SHA, + LockedFD + ) - + class TestUtils(TestBase): - def test_basics(self): - assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA - assert len(to_bin_sha(NULL_HEX_SHA)) == 20 - assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA - - def _cmp_contents(self, file_path, data): - # raise if data from file at file_path - # does not match data string - fp = open(file_path, "rb") - try: - assert fp.read() == data - finally: - fp.close() - - def test_lockedfd(self): - my_file = tempfile.mktemp() - orig_data = "hello" - new_data = "world" - my_file_fp = open(my_file, "wb") - my_file_fp.write(orig_data) - my_file_fp.close() - - try: - lfd = LockedFD(my_file) - lockfilepath = lfd._lockfilepath() - - # cannot end before it was started - self.failUnlessRaises(AssertionError, lfd.rollback) - self.failUnlessRaises(AssertionError, lfd.commit) - - # open for writing - assert not os.path.isfile(lockfilepath) - wfd = lfd.open(write=True) - assert lfd._fd is wfd - assert os.path.isfile(lockfilepath) - - # write data and fail - os.write(wfd, new_data) - lfd.rollback() - assert lfd._fd is None - self._cmp_contents(my_file, orig_data) - assert not os.path.isfile(lockfilepath) - - # additional call doesnt fail - lfd.commit() - lfd.rollback() - - # test reading - lfd = LockedFD(my_file) - rfd = lfd.open(write=False) - assert os.read(rfd, len(orig_data)) == orig_data - - assert os.path.isfile(lockfilepath) - # deletion rolls back - del(lfd) - assert not os.path.isfile(lockfilepath) - - - # write data - concurrently - lfd = LockedFD(my_file) - olfd = LockedFD(my_file) - assert not os.path.isfile(lockfilepath) - wfdstream = lfd.open(write=True, stream=True) # this time as stream - assert os.path.isfile(lockfilepath) - # another one fails - self.failUnlessRaises(IOError, olfd.open) - - wfdstream.write(new_data) - lfd.commit() - assert not os.path.isfile(lockfilepath) - self._cmp_contents(my_file, new_data) - - # could test automatic _end_writing on destruction - finally: - os.remove(my_file) - # END final cleanup - - # try non-existing file for reading - lfd = LockedFD(tempfile.mktemp()) - try: - lfd.open(write=False) - except OSError: - assert not os.path.exists(lfd._lockfilepath()) - else: - self.fail("expected OSError") - # END handle exceptions + def test_basics(self): + assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA + assert len(to_bin_sha(NULL_HEX_SHA)) == 20 + assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA + + def _cmp_contents(self, file_path, data): + # raise if data from file at file_path + # does not match data string + fp = open(file_path, "rb") + try: + assert fp.read() == data + finally: + fp.close() + + def test_lockedfd(self): + my_file = tempfile.mktemp() + orig_data = "hello" + new_data = "world" + my_file_fp = open(my_file, "wb") + my_file_fp.write(orig_data) + my_file_fp.close() + + try: + lfd = LockedFD(my_file) + lockfilepath = lfd._lockfilepath() + + # cannot end before it was started + self.failUnlessRaises(AssertionError, lfd.rollback) + self.failUnlessRaises(AssertionError, lfd.commit) + + # open for writing + assert not os.path.isfile(lockfilepath) + wfd = lfd.open(write=True) + assert lfd._fd is wfd + assert os.path.isfile(lockfilepath) + + # write data and fail + os.write(wfd, new_data) + lfd.rollback() + assert lfd._fd is None + self._cmp_contents(my_file, orig_data) + assert not os.path.isfile(lockfilepath) + + # additional call doesnt fail + lfd.commit() + lfd.rollback() + + # test reading + lfd = LockedFD(my_file) + rfd = lfd.open(write=False) + assert os.read(rfd, len(orig_data)) == orig_data + + assert os.path.isfile(lockfilepath) + # deletion rolls back + del(lfd) + assert not os.path.isfile(lockfilepath) + + + # write data - concurrently + lfd = LockedFD(my_file) + olfd = LockedFD(my_file) + assert not os.path.isfile(lockfilepath) + wfdstream = lfd.open(write=True, stream=True) # this time as stream + assert os.path.isfile(lockfilepath) + # another one fails + self.failUnlessRaises(IOError, olfd.open) + + wfdstream.write(new_data) + lfd.commit() + assert not os.path.isfile(lockfilepath) + self._cmp_contents(my_file, new_data) + + # could test automatic _end_writing on destruction + finally: + os.remove(my_file) + # END final cleanup + + # try non-existing file for reading + lfd = LockedFD(tempfile.mktemp()) + try: + lfd.open(write=False) + except OSError: + assert not os.path.exists(lfd._lockfilepath()) + else: + self.fail("expected OSError") + # END handle exceptions diff --git a/gitdb/util.py b/gitdb/util.py index 013f5fc78..1662b662d 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -13,28 +13,28 @@ # in py 2.4, StringIO is only StringI, without write support. # Hence we must use the python implementation for this if sys.version_info[1] < 5: - from StringIO import StringIO + from StringIO import StringIO # END handle python 2.4 try: - import async.mod.zlib as zlib + import async.mod.zlib as zlib except ImportError: - import zlib + import zlib # END try async zlib from async import ThreadPool from smmap import ( - StaticWindowMapManager, - SlidingWindowMapManager, - SlidingWindowMapBuffer - ) + StaticWindowMapManager, + SlidingWindowMapManager, + SlidingWindowMapBuffer + ) # initialize our global memory manager instance # Use it to free cached (and unused) resources. if sys.version_info[1] < 6: - mman = StaticWindowMapManager() + mman = StaticWindowMapManager() else: - mman = SlidingWindowMapManager() + mman = SlidingWindowMapManager() #END handle mman try: @@ -43,19 +43,19 @@ import sha try: - from struct import unpack_from + from struct import unpack_from except ImportError: - from struct import unpack, calcsize - __calcsize_cache = dict() - def unpack_from(fmt, data, offset=0): - try: - size = __calcsize_cache[fmt] - except KeyError: - size = calcsize(fmt) - __calcsize_cache[fmt] = size - # END exception handling - return unpack(fmt, data[offset : offset + size]) - # END own unpack_from implementation + from struct import unpack, calcsize + __calcsize_cache = dict() + def unpack_from(fmt, data, offset=0): + try: + size = __calcsize_cache[fmt] + except KeyError: + size = calcsize(fmt) + __calcsize_cache[fmt] = size + # END exception handling + return unpack(fmt, data[offset : offset + size]) + # END own unpack_from implementation #{ Globals @@ -100,25 +100,25 @@ def unpack_from(fmt, data, offset=0): #{ compatibility stuff ... class _RandomAccessStringIO(object): - """Wrapper to provide required functionality in case memory maps cannot or may - not be used. This is only really required in python 2.4""" - __slots__ = '_sio' - - def __init__(self, buf=''): - self._sio = StringIO(buf) - - def __getattr__(self, attr): - return getattr(self._sio, attr) - - def __len__(self): - return len(self.getvalue()) - - def __getitem__(self, i): - return self.getvalue()[i] - - def __getslice__(self, start, end): - return self.getvalue()[start:end] - + """Wrapper to provide required functionality in case memory maps cannot or may + not be used. This is only really required in python 2.4""" + __slots__ = '_sio' + + def __init__(self, buf=''): + self._sio = StringIO(buf) + + def __getattr__(self, attr): + return getattr(self._sio, attr) + + def __len__(self): + return len(self.getvalue()) + + def __getitem__(self, i): + return self.getvalue()[i] + + def __getslice__(self, start, end): + return self.getvalue()[start:end] + #} END compatibility stuff ... #{ Routines @@ -134,85 +134,85 @@ def make_sha(source=''): return sha1 def allocate_memory(size): - """:return: a file-protocol accessible memory block of the given size""" - if size == 0: - return _RandomAccessStringIO('') - # END handle empty chunks gracefully - - try: - return mmap.mmap(-1, size) # read-write by default - except EnvironmentError: - # setup real memory instead - # this of course may fail if the amount of memory is not available in - # one chunk - would only be the case in python 2.4, being more likely on - # 32 bit systems. - return _RandomAccessStringIO("\0"*size) - # END handle memory allocation - + """:return: a file-protocol accessible memory block of the given size""" + if size == 0: + return _RandomAccessStringIO('') + # END handle empty chunks gracefully + + try: + return mmap.mmap(-1, size) # read-write by default + except EnvironmentError: + # setup real memory instead + # this of course may fail if the amount of memory is not available in + # one chunk - would only be the case in python 2.4, being more likely on + # 32 bit systems. + return _RandomAccessStringIO("\0"*size) + # END handle memory allocation + def file_contents_ro(fd, stream=False, allow_mmap=True): - """:return: read-only contents of the file represented by the file descriptor fd - - :param fd: file descriptor opened for reading - :param stream: if False, random access is provided, otherwise the stream interface - is provided. - :param allow_mmap: if True, its allowed to map the contents into memory, which - allows large files to be handled and accessed efficiently. The file-descriptor - will change its position if this is False""" - try: - if allow_mmap: - # supports stream and random access - try: - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - except EnvironmentError: - # python 2.4 issue, 0 wants to be the actual size - return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) - # END handle python 2.4 - except OSError: - pass - # END exception handling - - # read manully - contents = os.read(fd, os.fstat(fd).st_size) - if stream: - return _RandomAccessStringIO(contents) - return contents - + """:return: read-only contents of the file represented by the file descriptor fd + + :param fd: file descriptor opened for reading + :param stream: if False, random access is provided, otherwise the stream interface + is provided. + :param allow_mmap: if True, its allowed to map the contents into memory, which + allows large files to be handled and accessed efficiently. The file-descriptor + will change its position if this is False""" + try: + if allow_mmap: + # supports stream and random access + try: + return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + except EnvironmentError: + # python 2.4 issue, 0 wants to be the actual size + return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) + # END handle python 2.4 + except OSError: + pass + # END exception handling + + # read manully + contents = os.read(fd, os.fstat(fd).st_size) + if stream: + return _RandomAccessStringIO(contents) + return contents + def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): - """Get the file contents at filepath as fast as possible - - :return: random access compatible memory of the given filepath - :param stream: see ``file_contents_ro`` - :param allow_mmap: see ``file_contents_ro`` - :param flags: additional flags to pass to os.open - :raise OSError: If the file could not be opened - - **Note** for now we don't try to use O_NOATIME directly as the right value needs to be - shared per database in fact. It only makes a real difference for loose object - databases anyway, and they use it with the help of the ``flags`` parameter""" - fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) - try: - return file_contents_ro(fd, stream, allow_mmap) - finally: - close(fd) - # END assure file is closed - + """Get the file contents at filepath as fast as possible + + :return: random access compatible memory of the given filepath + :param stream: see ``file_contents_ro`` + :param allow_mmap: see ``file_contents_ro`` + :param flags: additional flags to pass to os.open + :raise OSError: If the file could not be opened + + **Note** for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object + databases anyway, and they use it with the help of the ``flags`` parameter""" + fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + try: + return file_contents_ro(fd, stream, allow_mmap) + finally: + close(fd) + # END assure file is closed + def sliding_ro_buffer(filepath, flags=0): - """ - :return: a buffer compatible object which uses our mapped memory manager internally - ready to read the whole given filepath""" - return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) - + """ + :return: a buffer compatible object which uses our mapped memory manager internally + ready to read the whole given filepath""" + return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) + def to_hex_sha(sha): - """:return: hexified version of sha""" - if len(sha) == 40: - return sha - return bin_to_hex(sha) - + """:return: hexified version of sha""" + if len(sha) == 40: + return sha + return bin_to_hex(sha) + def to_bin_sha(sha): - if len(sha) == 20: - return sha - return hex_to_bin(sha) + if len(sha) == 20: + return sha + return hex_to_bin(sha) #} END routines @@ -221,162 +221,162 @@ def to_bin_sha(sha): #{ Utilities class LazyMixin(object): - """ - Base class providing an interface to lazily retrieve attribute values upon - first access. If slots are used, memory will only be reserved once the attribute - is actually accessed and retrieved the first time. All future accesses will - return the cached value as stored in the Instance's dict or slot. - """ - - __slots__ = tuple() - - def __getattr__(self, attr): - """ - Whenever an attribute is requested that we do not know, we allow it - to be created and set. Next time the same attribute is reqeusted, it is simply - returned from our dict/slots. """ - self._set_cache_(attr) - # will raise in case the cache was not created - return object.__getattribute__(self, attr) - - def _set_cache_(self, attr): - """ - This method should be overridden in the derived class. - It should check whether the attribute named by attr can be created - and cached. Do nothing if you do not know the attribute or call your subclass - - The derived class may create as many additional attributes as it deems - necessary in case a git command returns more information than represented - in the single attribute.""" - pass - - + """ + Base class providing an interface to lazily retrieve attribute values upon + first access. If slots are used, memory will only be reserved once the attribute + is actually accessed and retrieved the first time. All future accesses will + return the cached value as stored in the Instance's dict or slot. + """ + + __slots__ = tuple() + + def __getattr__(self, attr): + """ + Whenever an attribute is requested that we do not know, we allow it + to be created and set. Next time the same attribute is reqeusted, it is simply + returned from our dict/slots. """ + self._set_cache_(attr) + # will raise in case the cache was not created + return object.__getattribute__(self, attr) + + def _set_cache_(self, attr): + """ + This method should be overridden in the derived class. + It should check whether the attribute named by attr can be created + and cached. Do nothing if you do not know the attribute or call your subclass + + The derived class may create as many additional attributes as it deems + necessary in case a git command returns more information than represented + in the single attribute.""" + pass + + class LockedFD(object): - """ - This class facilitates a safe read and write operation to a file on disk. - If we write to 'file', we obtain a lock file at 'file.lock' and write to - that instead. If we succeed, the lock file will be renamed to overwrite - the original file. - - When reading, we obtain a lock file, but to prevent other writers from - succeeding while we are reading the file. - - This type handles error correctly in that it will assure a consistent state - on destruction. - - **note** with this setup, parallel reading is not possible""" - __slots__ = ("_filepath", '_fd', '_write') - - def __init__(self, filepath): - """Initialize an instance with the givne filepath""" - self._filepath = filepath - self._fd = None - self._write = None # if True, we write a file - - def __del__(self): - # will do nothing if the file descriptor is already closed - if self._fd is not None: - self.rollback() - - def _lockfilepath(self): - return "%s.lock" % self._filepath - - def open(self, write=False, stream=False): - """ - Open the file descriptor for reading or writing, both in binary mode. - - :param write: if True, the file descriptor will be opened for writing. Other - wise it will be opened read-only. - :param stream: if True, the file descriptor will be wrapped into a simple stream - object which supports only reading or writing - :return: fd to read from or write to. It is still maintained by this instance - and must not be closed directly - :raise IOError: if the lock could not be retrieved - :raise OSError: If the actual file could not be opened for reading - - **note** must only be called once""" - if self._write is not None: - raise AssertionError("Called %s multiple times" % self.open) - - self._write = write - - # try to open the lock file - binary = getattr(os, 'O_BINARY', 0) - lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary - try: - fd = os.open(self._lockfilepath(), lockmode, 0600) - if not write: - os.close(fd) - else: - self._fd = fd - # END handle file descriptor - except OSError: - raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) - # END handle lock retrieval - - # open actual file if required - if self._fd is None: - # we could specify exlusive here, as we obtained the lock anyway - try: - self._fd = os.open(self._filepath, os.O_RDONLY | binary) - except: - # assure we release our lockfile - os.remove(self._lockfilepath()) - raise - # END handle lockfile - # END open descriptor for reading - - if stream: - # need delayed import - from stream import FDStream - return FDStream(self._fd) - else: - return self._fd - # END handle stream - - def commit(self): - """When done writing, call this function to commit your changes into the - actual file. - The file descriptor will be closed, and the lockfile handled. - - **Note** can be called multiple times""" - self._end_writing(successful=True) - - def rollback(self): - """Abort your operation without any changes. The file descriptor will be - closed, and the lock released. - - **Note** can be called multiple times""" - self._end_writing(successful=False) - - def _end_writing(self, successful=True): - """Handle the lock according to the write mode """ - if self._write is None: - raise AssertionError("Cannot end operation if it wasn't started yet") - - if self._fd is None: - return - - os.close(self._fd) - self._fd = None - - lockfile = self._lockfilepath() - if self._write and successful: - # on windows, rename does not silently overwrite the existing one - if sys.platform == "win32": - if isfile(self._filepath): - os.remove(self._filepath) - # END remove if exists - # END win32 special handling - os.rename(lockfile, self._filepath) - - # assure others can at least read the file - the tmpfile left it at rw-- - # We may also write that file, on windows that boils down to a remove- - # protection as well - chmod(self._filepath, 0644) - else: - # just delete the file so far, we failed - os.remove(lockfile) - # END successful handling + """ + This class facilitates a safe read and write operation to a file on disk. + If we write to 'file', we obtain a lock file at 'file.lock' and write to + that instead. If we succeed, the lock file will be renamed to overwrite + the original file. + + When reading, we obtain a lock file, but to prevent other writers from + succeeding while we are reading the file. + + This type handles error correctly in that it will assure a consistent state + on destruction. + + **note** with this setup, parallel reading is not possible""" + __slots__ = ("_filepath", '_fd', '_write') + + def __init__(self, filepath): + """Initialize an instance with the givne filepath""" + self._filepath = filepath + self._fd = None + self._write = None # if True, we write a file + + def __del__(self): + # will do nothing if the file descriptor is already closed + if self._fd is not None: + self.rollback() + + def _lockfilepath(self): + return "%s.lock" % self._filepath + + def open(self, write=False, stream=False): + """ + Open the file descriptor for reading or writing, both in binary mode. + + :param write: if True, the file descriptor will be opened for writing. Other + wise it will be opened read-only. + :param stream: if True, the file descriptor will be wrapped into a simple stream + object which supports only reading or writing + :return: fd to read from or write to. It is still maintained by this instance + and must not be closed directly + :raise IOError: if the lock could not be retrieved + :raise OSError: If the actual file could not be opened for reading + + **note** must only be called once""" + if self._write is not None: + raise AssertionError("Called %s multiple times" % self.open) + + self._write = write + + # try to open the lock file + binary = getattr(os, 'O_BINARY', 0) + lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary + try: + fd = os.open(self._lockfilepath(), lockmode, 0600) + if not write: + os.close(fd) + else: + self._fd = fd + # END handle file descriptor + except OSError: + raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) + # END handle lock retrieval + + # open actual file if required + if self._fd is None: + # we could specify exlusive here, as we obtained the lock anyway + try: + self._fd = os.open(self._filepath, os.O_RDONLY | binary) + except: + # assure we release our lockfile + os.remove(self._lockfilepath()) + raise + # END handle lockfile + # END open descriptor for reading + + if stream: + # need delayed import + from stream import FDStream + return FDStream(self._fd) + else: + return self._fd + # END handle stream + + def commit(self): + """When done writing, call this function to commit your changes into the + actual file. + The file descriptor will be closed, and the lockfile handled. + + **Note** can be called multiple times""" + self._end_writing(successful=True) + + def rollback(self): + """Abort your operation without any changes. The file descriptor will be + closed, and the lock released. + + **Note** can be called multiple times""" + self._end_writing(successful=False) + + def _end_writing(self, successful=True): + """Handle the lock according to the write mode """ + if self._write is None: + raise AssertionError("Cannot end operation if it wasn't started yet") + + if self._fd is None: + return + + os.close(self._fd) + self._fd = None + + lockfile = self._lockfilepath() + if self._write and successful: + # on windows, rename does not silently overwrite the existing one + if sys.platform == "win32": + if isfile(self._filepath): + os.remove(self._filepath) + # END remove if exists + # END win32 special handling + os.rename(lockfile, self._filepath) + + # assure others can at least read the file - the tmpfile left it at rw-- + # We may also write that file, on windows that boils down to a remove- + # protection as well + chmod(self._filepath, 0644) + else: + # just delete the file so far, we failed + os.remove(lockfile) + # END successful handling #} END utilities From 6099ac8e04a8f734555efe4e78ad766d9a6fb70a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:23:09 +0200 Subject: [PATCH 0205/1392] Added travis CI support --- .travis.yml | 8 ++++++++ README.rst | 8 +++++--- 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..e7d5214c1 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: python +python: + - "2.5" + - "2.6" + - "2.7" + - "pypy" + +script: nosetests diff --git a/README.rst b/README.rst index 30bff0ded..84feb373c 100644 --- a/README.rst +++ b/README.rst @@ -9,6 +9,8 @@ Although memory maps have many advantages, they represent a very limited system Overview ######## +.. image:: https://travis-ci.org/Byron/smmap.svg?branch=master :target: https://travis-ci.org/Byron/smmap + Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. @@ -58,17 +60,17 @@ The project is home on github at `https://github.com/Byron/smmap Date: Sun, 4 May 2014 16:29:43 +0200 Subject: [PATCH 0206/1392] pypy deactivated, it doesn't yet work --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e7d5214c1..563797656 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,9 @@ language: python python: + - "2.4" - "2.5" - "2.6" - "2.7" - - "pypy" + # - "pypy" - no getrefcount script: nosetests From 616e9ceaf917e4d8f3cf2c145401b8069ce307dd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:33:55 +0200 Subject: [PATCH 0207/1392] Oh, travis doesn't support older python versions, fair enough --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 563797656..fdba549d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: python python: - - "2.4" - - "2.5" - "2.6" - "2.7" # - "pypy" - no getrefcount From a4eddfe72b442666ba3acdf89929e9b65de45f45 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:38:55 +0200 Subject: [PATCH 0208/1392] Added initial configuration of gitdb --- README.rst | 3 +++ gitdb/.travis.yml | 7 +++++++ gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- 4 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 gitdb/.travis.yml diff --git a/README.rst b/README.rst index 753eb701c..68f71ff61 100644 --- a/README.rst +++ b/README.rst @@ -32,6 +32,9 @@ http://groups.google.com/group/git-python ISSUE TRACKER ============= + +.. image:: https://travis-ci.org/Byron/gitdb.svg?branch=master :target: https://travis-ci.org/Byron/gitdb + https://github.com/gitpython-developers/gitdb/issues LICENSE diff --git a/gitdb/.travis.yml b/gitdb/.travis.yml new file mode 100644 index 000000000..5c8791d90 --- /dev/null +++ b/gitdb/.travis.yml @@ -0,0 +1,7 @@ +language: python +python: + - "2.6" + - "2.7" + # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) + +script: nosetests diff --git a/gitdb/ext/async b/gitdb/ext/async index 571412931..90326fb86 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 571412931829200aff06a44b9c5524e122e524e9 +Subproject commit 90326fb867f94b193c277b07b23e364047e1ed28 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 1b3ab5598..616e9ceaf 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 1b3ab5598e93369282502d049d64cb2ca12839cb +Subproject commit 616e9ceaf917e4d8f3cf2c145401b8069ce307dd From e2c94bf6983b378f99db175cd7810986cf338c32 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:43:26 +0200 Subject: [PATCH 0209/1392] argh, one level too low, didn't see it in sublime --- gitdb/.travis.yml => .travis.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename gitdb/.travis.yml => .travis.yml (100%) diff --git a/gitdb/.travis.yml b/.travis.yml similarity index 100% rename from gitdb/.travis.yml rename to .travis.yml From 39de1127459b73b862f2b779bb4565ad6b4bd625 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 May 2014 16:44:48 +0200 Subject: [PATCH 0210/1392] Fixed travis build status url --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 68f71ff61..0fc0534e6 100644 --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ http://groups.google.com/group/git-python ISSUE TRACKER ============= -.. image:: https://travis-ci.org/Byron/gitdb.svg?branch=master :target: https://travis-ci.org/Byron/gitdb +.. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master :target: https://travis-ci.org/gitpython-developers/gitdb https://github.com/gitpython-developers/gitdb/issues From cf1cf469f315df00ce7b9d47693008cd2fa1b56a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 May 2014 09:31:15 +0200 Subject: [PATCH 0211/1392] Get starten on py3.3 compatability This is likely to fail, but lets see. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fdba549d4..d8640a08e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "2.6" - "2.7" + - "3.3" # - "pypy" - no getrefcount script: nosetests From 32d34eba0412770c382a1428a0ebb221dbba4187 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 May 2014 09:55:32 +0200 Subject: [PATCH 0212/1392] Test with python 3.3 as well Have to start making them compatible at some point. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5c8791d90..ff263f20d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "2.6" - "2.7" + - "3.3" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) script: nosetests From 2d3b5a303a65d6f80b84e63a1b3cf4b670c81f9a Mon Sep 17 00:00:00 2001 From: David Black Date: Fri, 16 May 2014 15:38:03 +1000 Subject: [PATCH 0213/1392] Initial work for supporting python 3 (>= 3.3). Signed-off-by: David Black --- smmap/__init__.py | 4 ++-- smmap/buf.py | 8 ++++---- smmap/mman.py | 20 +++++++++++--------- smmap/test/test_buf.py | 6 +++--- smmap/test/test_mman.py | 8 ++++---- smmap/test/test_tutorial.py | 2 +- smmap/test/test_util.py | 8 ++++---- smmap/util.py | 13 ++++++++++--- 8 files changed, 39 insertions(+), 30 deletions(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index a10cd5c99..879ebea24 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -7,5 +7,5 @@ __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience -from mman import * -from buf import * +from .mman import * +from .buf import * diff --git a/smmap/buf.py b/smmap/buf.py index 255c6b54d..3917ee8be 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,5 +1,5 @@ """Module with a simple buffer implementation using the memory manager""" -from mman import WindowCursor +from .mman import WindowCursor import sys @@ -21,7 +21,7 @@ class SlidingWindowMapBuffer(object): ) - def __init__(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + def __init__(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0): """Initalize the instance to operate on the given cursor. :param cursor: if not None, the associated cursor to the file you want to access If None, you have call begin_access before using the buffer and provide a cursor @@ -61,7 +61,7 @@ def __getslice__(self, i, j): assert c.is_valid() if i < 0: i = self._size + i - if j == sys.maxint: + if j == sys.maxsize: j = self._size if j < 0: j = self._size + j @@ -86,7 +86,7 @@ def __getslice__(self, i, j): # END fast or slow path #{ Interface - def begin_access(self, cursor = None, offset = 0, size = sys.maxint, flags = 0): + def begin_access(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0): """Call this before the first use of this instance. The method was already called by the constructor in case sufficient information was provided. diff --git a/smmap/mman.py b/smmap/mman.py index 97c42c5bb..9cc251f0f 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,15 +1,17 @@ """Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" -from util import ( +from .util import ( MapWindow, MapRegion, MapRegionList, is_64_bit, - align_to_mmap + align_to_mmap, + string_types, ) from weakref import ref import sys from sys import getrefcount +from functools import reduce __all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"] #{ Utilities @@ -218,7 +220,7 @@ def fd(self): **Note:** it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" - if isinstance(self._rlist.path_or_fd(), basestring): + if isinstance(self._rlist.path_or_fd(), string_types()): raise ValueError("File descriptor queried although mapping was generated from path") #END handle type return self._rlist.path_or_fd() @@ -256,7 +258,7 @@ class StaticWindowMapManager(object): _MB_in_bytes = 1024 * 1024 - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint): + def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxsize): """initialize the manager with the given parameters. :param window_size: if -1, a default window size will be chosen depending on the operating system's architechture. It will internally be quantified to a multiple of the page size @@ -306,7 +308,7 @@ def _collect_lru_region(self, size): while (size == 0) or (self._memory_size + size > self._max_memory_size): lru_region = None lru_list = None - for regions in self._fdict.itervalues(): + for regions in self._fdict.values(): for region in regions: # check client count - consider that we keep one reference ourselves ! if (region.client_count()-2 == 0 and @@ -343,7 +345,7 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): r = a[0] else: try: - r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxint, flags) + r = self.MapRegionCls(a.path_or_fd(), 0, sys.maxsize, flags) except Exception: # apparently we are out of system resources or hit a limit # As many more operations are likely to fail in that condition ( @@ -405,7 +407,7 @@ def num_file_handles(self): def num_open_files(self): """Amount of opened files in the system""" - return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.itervalues() if len(rlist) > 0), 0) + return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) def window_size(self): """:return: size of each window when allocating new regions""" @@ -445,7 +447,7 @@ def force_map_handle_removal_win(self, base_path): #END early bailout num_closed = 0 - for path, rlist in self._fdict.iteritems(): + for path, rlist in self._fdict.items(): if path.startswith(base_path): for region in rlist: region._mf.close() @@ -473,7 +475,7 @@ class SlidingWindowMapManager(StaticWindowMapManager): __slots__ = tuple() - def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxint): + def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxsize): """Adjusts the default window size to -1""" super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 4bdcb76f5..d40da1479 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,4 +1,4 @@ -from lib import TestBase, FileCreator +from .lib import TestBase, FileCreator from smmap.mman import SlidingWindowMapManager, StaticWindowMapManager from smmap.buf import * @@ -22,8 +22,8 @@ def test_basics(self): # invalid paths fail upon construction c = man_optimal.make_cursor(fc.path) - self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor - self.failUnlessRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large + self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor + self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large buf = SlidingWindowMapBuffer() # can create uninitailized buffers assert buf.cursor() is None diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 46429a419..0929583c9 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,4 +1,4 @@ -from lib import TestBase, FileCreator +from .lib import TestBase, FileCreator from smmap.mman import * from smmap.mman import WindowCursor @@ -66,7 +66,7 @@ def test_memory_manager(self): man._collect_lru_region(10) # doesn't fail if we overallocate - assert man._collect_lru_region(sys.maxint) == 0 + assert man._collect_lru_region(sys.maxsize) == 0 # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") @@ -80,9 +80,9 @@ def test_memory_manager(self): assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] if isinstance(item, int): - self.failUnlessRaises(ValueError, c.path) + self.assertRaises(ValueError, c.path) else: - self.failUnlessRaises(ValueError, c.fd) + self.assertRaises(ValueError, c.fd) #END handle value error #END for each input os.close(fd) diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index 4e1a5764b..ad1a9c0b5 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -1,4 +1,4 @@ -from lib import TestBase +from .lib import TestBase class TestTutorial(TestBase): diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 2df0660be..a009bd9d2 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,4 +1,4 @@ -from lib import TestBase, FileCreator +from .lib import TestBase, FileCreator from smmap.util import * @@ -38,7 +38,7 @@ def test_window(self): assert wc.ofs == 1 and wc.size == maxsize # without maxsize - wc.extend_right_to(wr, sys.maxint) + wc.extend_right_to(wr, sys.maxsize) assert wc.ofs_end() == wr.ofs and wc.ofs == 1 # extend left @@ -46,7 +46,7 @@ def test_window(self): wr.extend_left_to(wc2, maxsize) assert wr.size == maxsize - wr.extend_left_to(wc2, sys.maxint) + wr.extend_left_to(wc2, sys.maxsize) assert wr.ofs == wc2.ofs_end() wc.align() @@ -68,7 +68,7 @@ def test_region(self): assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) - assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxint) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) # with the values we have, this test only works on windows where an alignment # size of 4096 is assumed. # We only test on linux as it is inconsitent between the python versions diff --git a/smmap/util.py b/smmap/util.py index c6710b3fe..fee9ad591 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -19,6 +19,13 @@ #{ Utilities +def string_types(): + if sys.version_info[0] >= 3: + return str + else: + return basestring + + def align_to_mmap(num, round_up): """ Align the given integer number to the closest page offset, which usually is 4096 bytes. @@ -34,7 +41,7 @@ def align_to_mmap(num, round_up): def is_64_bit(): """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" - return sys.maxint > (1<<32) - 1 + return sys.maxsize > (1<<32) - 1 #}END utilities @@ -154,7 +161,7 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): self._mfb = buffer(self._mf, ofs, self._size) #END handle buffer wrapping finally: - if isinstance(path_or_fd, basestring): + if isinstance(path_or_fd, string_types()): os.close(fd) #END only close it if we opened it #END close file handle @@ -258,7 +265,7 @@ def path_or_fd(self): def file_size(self): """:return: size of file we manager""" if self._file_size is None: - if isinstance(self._path_or_fd, basestring): + if isinstance(self._path_or_fd, string_types()): self._file_size = os.stat(self._path_or_fd).st_size else: self._file_size = os.fstat(self._path_or_fd).st_size From 575f4a895ea525ecfedbfc1dc4c6f032ad92ccdc Mon Sep 17 00:00:00 2001 From: David Black Date: Fri, 16 May 2014 16:35:20 +1000 Subject: [PATCH 0214/1392] instead of writing a string to a buffer api (in python 3) write a buffer. Signed-off-by: David Black --- smmap/test/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 21e6c5a09..01f6cc918 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -22,7 +22,7 @@ def __init__(self, size, prefix=''): fp = open(self._path, "wb") fp.seek(size-1) - fp.write('1') + fp.write(b'1') fp.close() assert os.path.getsize(self.path) == size From ef3b36e6ae20e7bd6c31c71623a5e0019ba9eec9 Mon Sep 17 00:00:00 2001 From: David Black Date: Fri, 16 May 2014 16:36:47 +1000 Subject: [PATCH 0215/1392] _need_compat_layer is not required in python 3. Signed-off-by: David Black --- smmap/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/util.py b/smmap/util.py index fee9ad591..f4ecd228c 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -104,7 +104,7 @@ class MapRegion(object): '_size', # cached size of our memory map '__weakref__' ] - _need_compat_layer = sys.version_info[1] < 6 + _need_compat_layer = sys.version_info[0] < 3 and sys.version_info[1] < 6 if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset From afcf97e6cd6584c97ccfe3de57bdc58df0ba9f8a Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Fri, 16 May 2014 08:31:09 -0700 Subject: [PATCH 0216/1392] Add tox.ini; make .travis.yml use it --- .gitignore | 1 + .travis.yml | 16 +++++++++------- tox.ini | 11 +++++++++++ 3 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 6cfb58df1..11852be02 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ build/ coverage dist/ MANIFEST +.tox diff --git a/.travis.yml b/.travis.yml index d8640a08e..91f43283b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,10 @@ language: python -python: - - "2.6" - - "2.7" - - "3.3" - # - "pypy" - no getrefcount - -script: nosetests +env: + - TOXENV=py26 + - TOXENV=py27 + - TOXENV=py33 + - TOXENV=py34 +install: + - pip install tox +script: + - tox diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..80190d0a3 --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py26, py27, py33, py34 + +[testenv] +commands = nosetests +deps = nose From f0680101739da3435bbae0139765bb5ce65bcb92 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 19 May 2014 23:51:50 +0200 Subject: [PATCH 0217/1392] Added coverage reporting. In the process, I removed tox as it made things so much more complex for me. --- .coveragerc | 10 ++++++ .gitignore | 1 + .travis.yml | 15 ++++----- README.rst => README.md | 70 ++++++++++++++++++++++------------------- tox.ini | 11 ------- 5 files changed, 57 insertions(+), 50 deletions(-) create mode 100644 .coveragerc rename README.rst => README.md (71%) delete mode 100644 tox.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..e61d27ba9 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,10 @@ +[run] +source = smmap + +; to make nosetests happy +[report] +omit = + */yaml* + */tests/* + */python?.?/* + */site-packages/nose/* \ No newline at end of file diff --git a/.gitignore b/.gitignore index 11852be02..2081aafd2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ build/ .coverage coverage +cover/ dist/ MANIFEST .tox diff --git a/.travis.yml b/.travis.yml index 91f43283b..c63e5e325 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,11 @@ language: python -env: - - TOXENV=py26 - - TOXENV=py27 - - TOXENV=py33 - - TOXENV=py34 +python: + - "2.6" + - "2.7" + - "3.3" install: - - pip install tox + - pip install coveralls script: - - tox + - nosetests --with-coverage +after_success: + - coveralls diff --git a/README.rst b/README.md similarity index 71% rename from README.rst rename to README.md index 84feb373c..c056ed1c1 100644 --- a/README.rst +++ b/README.md @@ -1,15 +1,15 @@ -########### -Motivation -########### +## Motivation + When reading from many possibly large files in a fashion similar to random access, it is usually the fastest and most efficient to use memory maps. Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. -######## -Overview -######## -.. image:: https://travis-ci.org/Byron/smmap.svg?branch=master :target: https://travis-ci.org/Byron/smmap + +## Overview + +[![Build Status](https://travis-ci.org/Byron/smmap.svg?branch=master)](https://travis-ci.org/Byron/smmap) +[![Coverage Status](https://coveralls.io/repos/Byron/smmap/badge.png)](https://coveralls.io/r/Byron/smmap) Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. @@ -21,42 +21,49 @@ Although the library can be used most efficiently with its native interface, a B For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. -############# -Prerequisites -############# + + +## Prerequisites + * Python 2.4, 2.5 or 2.6 * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. -########### -Limitations -########### + + +## Limitations + * The memory access is read-only by design. * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. * It wasn't tested on python 2.7 and 3.x. -################ -Installing smmap -################ -Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the `setuptools`_ or `pip`_ respectively:: + +## Installing smmap + +Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the [setuptools](http://peak.telecommunity.com/DevCenter/setuptools) or [pip](http://www.pip-installer.org/en/latest) respectively: - $ easy_install smmap - # or - $ pip install smmap +```bash +$ easy_install smmap +# or +$ pip install smmap +``` As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. -If you have downloaded the source archive, the package can be installed by running the ``setup.py`` script:: +If you have downloaded the source archive, the package can be installed by running the `setup.py` script: - $ python setup.py install +```bash +$ python setup.py install +``` + +It is advised to have a look at the **Usage Guide** for a brief introduction on the different database implementations. + -It is advised to have a look at the :ref:`Usage Guide ` for a brief introduction on the different database implementations. -################## -Homepage and Links -################## -The project is home on github at `https://github.com/Byron/smmap `_. +## Homepage and Links + +The project is home on github at https://github.com/Byron/smmap . The latest source can be cloned from github as well: @@ -72,10 +79,9 @@ Issues can be filed on github: * https://github.com/Byron/smmap/issues -################### -License Information -################### + + +## License Information + *smmap* is licensed under the New BSD License. -.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools -.. _pip: http://www.pip-installer.org/en/latest/ diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 80190d0a3..000000000 --- a/tox.ini +++ /dev/null @@ -1,11 +0,0 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - -[tox] -envlist = py26, py27, py33, py34 - -[testenv] -commands = nosetests -deps = nose From 403ad7816e2c88be78606cfee2d9944f535e7ed7 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Fri, 13 Jun 2014 22:24:23 -0700 Subject: [PATCH 0218/1392] Delay importing sys.getrefcount until needed This makes it possibly to at least install on PyPy Addresses: GH-4 ("pypy compatibility") --- smmap/mman.py | 1 - smmap/util.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 9cc251f0f..637a2844d 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -10,7 +10,6 @@ from weakref import ref import sys -from sys import getrefcount from functools import reduce __all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"] diff --git a/smmap/util.py b/smmap/util.py index f4ecd228c..ec86cbf16 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -12,8 +12,6 @@ from mmap import PAGESIZE as ALLOCATIONGRANULARITY #END handle pythons missing quality assurance -from sys import getrefcount - __all__ = [ "align_to_mmap", "is_64_bit", "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] @@ -210,6 +208,7 @@ def includes_ofs(self, ofs): def client_count(self): """:return: number of clients currently using this region""" + from sys import getrefcount # -1: self on stack, -1 self in this method, -1 self in getrefcount return getrefcount(self)-3 @@ -256,6 +255,7 @@ def __init__(self, path_or_fd): def client_count(self): """:return: amount of clients which hold a reference to this instance""" + from sys import getrefcount return getrefcount(self)-3 def path_or_fd(self): From f3d3502bef512ed48581f69fee9b655a91e06db8 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Sun, 15 Jun 2014 23:33:04 -0700 Subject: [PATCH 0219/1392] Change / to // (integer division) in several places This fixes a bunch of bugs and test failures in Python 3, which uses "true division" for / --- smmap/test/test_mman.py | 6 +++--- smmap/test/test_util.py | 2 +- smmap/util.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 0929583c9..e0516b21c 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -95,8 +95,8 @@ def test_memman_operation(self): fd = os.open(fc.path, os.O_RDONLY) max_num_handles = 15 #small_size = - for mtype, args in ( (StaticWindowMapManager, (0, fc.size / 3, max_num_handles)), - (SlidingWindowMapManager, (fc.size / 100, fc.size / 3, max_num_handles)),): + for mtype, args in ( (StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), + (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): for item in (fc.path, fd): assert len(data) == fc.size @@ -110,7 +110,7 @@ def test_memman_operation(self): base_offset = 5000 # window size is 0 for static managers, hence size will be 0. We take that into consideration - size = man.window_size() / 2 + size = man.window_size() // 2 assert c.use_region(base_offset, size).is_valid() rr = c.region_ref() assert rr().client_count() == 2 # the manager and the cursor and us diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index a009bd9d2..8afba005e 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -54,7 +54,7 @@ def test_window(self): def test_region(self): fc = FileCreator(self.k_window_test_size, "window_test") - half_size = fc.size / 2 + half_size = fc.size // 2 rofs = align_to_mmap(4200, False) rfull = MapRegion(fc.path, 0, fc.size) rhalfofs = MapRegion(fc.path, rofs, fc.size) diff --git a/smmap/util.py b/smmap/util.py index ec86cbf16..0d8385db2 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -31,7 +31,7 @@ def align_to_mmap(num, round_up): :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" - res = (num / ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; + res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; if round_up and (res != num): res += ALLOCATIONGRANULARITY #END handle size From 707885bebe6cefa45dae7dd13dadac5b2b98d16d Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 06:41:25 -0700 Subject: [PATCH 0220/1392] .gitignore: Add *.egg-info --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2081aafd2..73b0b6188 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ cover/ dist/ MANIFEST .tox +*.egg-info From 23b79ea2838510028de43a6edb1b09246dca9a46 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 06:47:31 -0700 Subject: [PATCH 0221/1392] Add back tox.ini for tox This time with support for measuring coverage. Ability to test quickly across multiple Python versions is crucial for working on Python 3 compatibility... --- tox.ini | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..e0e196418 --- /dev/null +++ b/tox.ini @@ -0,0 +1,13 @@ +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py26, py27, py33, py34 + +[testenv] +commands = nosetests {posargs:--with-coverage --cover-package=smmap} +deps = + nose + nosexcover From 52fba74d78d3fcce53fb89c7b2c20b24197296a8 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 06:33:41 -0700 Subject: [PATCH 0222/1392] Deal with lack of `buffer` in py3 --- smmap/mman.py | 1 + smmap/test/test_tutorial.py | 1 + smmap/util.py | 11 ++++++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/smmap/mman.py b/smmap/mman.py index 637a2844d..ff32bbb51 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -6,6 +6,7 @@ is_64_bit, align_to_mmap, string_types, + buffer, ) from weakref import ref diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index ad1a9c0b5..ccc113b4f 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -44,6 +44,7 @@ def test_example(self): # its recommended not to create big slices when feeding the buffer # into consumers (e.g. struct or zlib). # Instead, either give the buffer directly, or use pythons buffer command. + from smmap.util import buffer buffer(c.buffer(), 1, 9) # first 9 bytes without copying them # you can query absolute offsets, and check whether an offset is included diff --git a/smmap/util.py b/smmap/util.py index 0d8385db2..54c6c45c3 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -12,11 +12,20 @@ from mmap import PAGESIZE as ALLOCATIONGRANULARITY #END handle pythons missing quality assurance -__all__ = [ "align_to_mmap", "is_64_bit", +__all__ = [ "align_to_mmap", "is_64_bit", "buffer", "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities +try: + # Python 2 + buffer = buffer +except NameError: + # Python 3 has no `buffer`; only `memoryview` + def buffer(obj, offset, size): + return memoryview(obj[offset:offset+size]) + + def string_types(): if sys.version_info[0] >= 3: return str From cf515928248b7c34cb73b6e20edbdadebd086f96 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 07:55:21 -0700 Subject: [PATCH 0223/1392] Fix 2 instances of "containnig" => "containing" --- smmap/mman.py | 2 +- smmap/util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index ff32bbb51..7cbb535bd 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,4 +1,4 @@ -"""Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" +"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" from .util import ( MapWindow, MapRegion, diff --git a/smmap/util.py b/smmap/util.py index 54c6c45c3..c37dfdd31 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -1,4 +1,4 @@ -"""Module containnig a memory memory manager which provides a sliding window on a number of memory mapped files""" +"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" import os import sys import mmap From c0b94c67b86b4250364a951f4865d714952f792a Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 08:54:30 -0700 Subject: [PATCH 0224/1392] .travis.yml: Allow py33 to fail, add py34, etc. --- .travis.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index c63e5e325..d02eb6f5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,17 @@ language: python python: - - "2.6" - - "2.7" - - "3.3" + - 2.6 + - 2.7 + - 3.3 + - 3.4 install: - pip install coveralls script: - nosetests --with-coverage after_success: - coveralls +matrix: + allow_failures: + - python: 3.3 + - python: 3.4 + fast_finish: true From 33e6314941572a93b8031b1feecded22ef1f5f3c Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 08:49:04 -0700 Subject: [PATCH 0225/1392] Use bytes() instead of str() bytes() is more accurate and is actually correct in Python 3, whereas str() is incorrect in Python 3, because it's a Unicode string. --- smmap/buf.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/smmap/buf.py b/smmap/buf.py index 3917ee8be..ba6f8ede7 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -5,6 +5,12 @@ __all__ = ["SlidingWindowMapBuffer"] +try: + bytes +except NameError: + bytes = str + + class SlidingWindowMapBuffer(object): """A buffer like object which allows direct byte-wise object and slicing into memory of a mapped file. The mapping is controlled by the provided cursor. @@ -73,7 +79,7 @@ def __getslice__(self, i, j): ofs = i # Keeping tokens in a list could possible be faster, but the list # overhead outweighs the benefits (tested) ! - md = str() + md = bytes() while l: c.use_region(ofs, l) assert c.is_valid() From 19917bdc3d30d7d3e6dba082ba132a86d0190b09 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 09:05:33 -0700 Subject: [PATCH 0226/1392] Fix typo: "optimial" => "optimal" --- smmap/test/test_buf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d40da1479..23a4fbbcd 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -76,7 +76,7 @@ def test_basics(self): for item in (fc.path, fd): for manager, man_id in ( (man_optimal, 'optimal'), (man_worst_case, 'worst case'), - (static_man, 'static optimial')): + (static_man, 'static optimal')): buf = SlidingWindowMapBuffer(manager.make_cursor(item)) assert manager.num_file_handles() == 1 for access_mode in range(2): # single, multi From 8fde5f3fb8711f866612c657d20c9d3e9cf59f41 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 09:02:30 -0700 Subject: [PATCH 0227/1392] Make __getitem__ handle slice for Python 3 Python 3 doesn't have __getslice__ instead it uses __getitem__ with a slice object. --- smmap/buf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/smmap/buf.py b/smmap/buf.py index ba6f8ede7..2f27d4d01 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -51,6 +51,8 @@ def __len__(self): return self._size def __getitem__(self, i): + if isinstance(i, slice): + return self.__getslice__(i.start or 0, i.stop or self._size) c = self._c assert c.is_valid() if i < 0: From 651dfa8a359200b66e686998af2640efb54f16ad Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 10:48:43 -0700 Subject: [PATCH 0228/1392] Change / to // (integer division) in test_buf.py This fixes (the last!) test failure in Python 3, which uses "true division" for / --- smmap/test/test_buf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 23a4fbbcd..15dfb8238 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -10,9 +10,10 @@ man_optimal = SlidingWindowMapManager() -man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size/100, - max_memory_size=TestBase.k_window_test_size/3, - max_open_handles=15) +man_worst_case = SlidingWindowMapManager( + window_size=TestBase.k_window_test_size // 100, + max_memory_size=TestBase.k_window_test_size // 3, + max_open_handles=15) static_man = StaticWindowMapManager() class TestBuf(TestBase): From f75dcbcf5d10a639c031dd706fa7fcfa1784eecb Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 11:09:54 -0700 Subject: [PATCH 0229/1392] .travis.yml: Stop allowing failures for py3{3,4} --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index d02eb6f5e..47cb41170 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,3 @@ script: - nosetests --with-coverage after_success: - coveralls -matrix: - allow_failures: - - python: 3.3 - - python: 3.4 - fast_finish: true From fbbb3090ea1ca4ac3042acea7633241b0388f0b3 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 11:31:48 -0700 Subject: [PATCH 0230/1392] setup.py: Add Python 3 (and Python 2) classifiers --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index 2e97e59c6..13d2063ab 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,12 @@ "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", ], long_description=long_description, ) From 55267119140f3828a24b4986600ed21a1808d6cc Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 16 Jun 2014 12:02:10 -0700 Subject: [PATCH 0231/1392] setup.cfg: Specify that wheel is universal --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 From d79c655a7bf281a39662f4a4562d76312b69515a Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 14:21:09 -0400 Subject: [PATCH 0232/1392] Update async to the Python 2 / 3 compatible version --- gitdb/ext/async | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/async b/gitdb/ext/async index 90326fb86..339024bfb 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 90326fb867f94b193c277b07b23e364047e1ed28 +Subproject commit 339024bfb1d0a2b091e63d7a7ea23a1c63189f5c From 72167492334b756ddeaa606274e9348a70734cdb Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 14:27:22 -0400 Subject: [PATCH 0233/1392] Update smmap to a Python 3 compatible version --- gitdb/ext/smmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 616e9ceaf..552671191 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 616e9ceaf917e4d8f3cf2c145401b8069ce307dd +Subproject commit 55267119140f3828a24b4986600ed21a1808d6cc From 1c6f4c19289732bd13507eba9e54c9d692957137 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 15:35:24 -0400 Subject: [PATCH 0234/1392] Automated PEP 8 fixes --- gitdb/__init__.py | 4 +- gitdb/base.py | 205 ++++++++++--------- gitdb/db/base.py | 162 ++++++++------- gitdb/db/git.py | 34 ++-- gitdb/db/loose.py | 103 +++++----- gitdb/db/mem.py | 56 +++-- gitdb/db/pack.py | 88 ++++---- gitdb/db/ref.py | 18 +- gitdb/exc.py | 10 +- gitdb/fun.py | 230 ++++++++++----------- gitdb/pack.py | 394 ++++++++++++++++++------------------ gitdb/stream.py | 318 +++++++++++++++-------------- gitdb/test/__init__.py | 2 +- gitdb/test/db/lib.py | 80 ++++---- gitdb/test/db/test_git.py | 22 +- gitdb/test/db/test_loose.py | 15 +- gitdb/test/db/test_mem.py | 18 +- gitdb/test/db/test_pack.py | 28 +-- gitdb/test/db/test_ref.py | 28 ++- gitdb/test/lib.py | 39 ++-- gitdb/test/test_base.py | 28 +-- gitdb/test/test_example.py | 26 +-- gitdb/test/test_pack.py | 102 +++++----- gitdb/test/test_stream.py | 51 +++-- gitdb/test/test_util.py | 43 ++-- gitdb/util.py | 131 ++++++------ 26 files changed, 1108 insertions(+), 1127 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index ff750d14c..847269a33 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -12,14 +12,14 @@ def _init_externals(): """Initialize external projects by putting them into the path""" for module in ('async', 'smmap'): sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) - + try: __import__(module) except ImportError: raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) #END verify import #END handel imports - + #} END initialization _init_externals() diff --git a/gitdb/base.py b/gitdb/base.py index bad5f7472..a673c2376 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -13,183 +13,183 @@ type_to_type_id_map ) -__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', +__all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', 'OStream', 'OPackStream', 'ODeltaPackStream', 'IStream', 'InvalidOInfo', 'InvalidOStream' ) #{ ODB Bases class OInfo(tuple): - """Carries information about an object in an ODB, provding information + """Carries information about an object in an ODB, provding information about the binary sha of the object, the type_string as well as the uncompressed size in bytes. - + It can be accessed using tuple notation and using attribute access notation:: - + assert dbi[0] == dbi.binsha assert dbi[1] == dbi.type assert dbi[2] == dbi.size - + The type is designed to be as lighteight as possible.""" __slots__ = tuple() - + def __new__(cls, sha, type, size): return tuple.__new__(cls, (sha, type, size)) - + def __init__(self, *args): tuple.__init__(self) - - #{ Interface + + #{ Interface @property def binsha(self): """:return: our sha as binary, 20 bytes""" return self[0] - + @property def hexsha(self): """:return: our sha, hex encoded, 40 bytes""" return bin_to_hex(self[0]) - + @property def type(self): return self[1] - + @property def type_id(self): return type_to_type_id_map[self[1]] - + @property def size(self): return self[2] #} END interface - - + + class OPackInfo(tuple): - """As OInfo, but provides a type_id property to retrieve the numerical type id, and + """As OInfo, but provides a type_id property to retrieve the numerical type id, and does not include a sha. - - Additionally, the pack_offset is the absolute offset into the packfile at which + + Additionally, the pack_offset is the absolute offset into the packfile at which all object information is located. The data_offset property points to the abosolute location in the pack at which that actual data stream can be found.""" __slots__ = tuple() - + def __new__(cls, packoffset, type, size): return tuple.__new__(cls, (packoffset,type, size)) - + def __init__(self, *args): tuple.__init__(self) - - #{ Interface - + + #{ Interface + @property def pack_offset(self): return self[0] - + @property def type(self): return type_id_to_type_map[self[1]] - + @property def type_id(self): return self[1] - + @property def size(self): return self[2] - + #} END interface - - + + class ODeltaPackInfo(OPackInfo): - """Adds delta specific information, - Either the 20 byte sha which points to some object in the database, + """Adds delta specific information, + Either the 20 byte sha which points to some object in the database, or the negative offset from the pack_offset, so that pack_offset - delta_info yields the pack offset of the base object""" __slots__ = tuple() - + def __new__(cls, packoffset, type, size, delta_info): return tuple.__new__(cls, (packoffset, type, size, delta_info)) - - #{ Interface + + #{ Interface @property def delta_info(self): return self[3] - #} END interface - - + #} END interface + + class OStream(OInfo): - """Base for object streams retrieved from the database, providing additional + """Base for object streams retrieved from the database, providing additional information about the stream. Generally, ODB streams are read-only as objects are immutable""" __slots__ = tuple() - + def __new__(cls, sha, type, size, stream, *args, **kwargs): """Helps with the initialization of subclasses""" return tuple.__new__(cls, (sha, type, size, stream)) - - + + def __init__(self, *args, **kwargs): tuple.__init__(self) - - #{ Stream Reader Interface - + + #{ Stream Reader Interface + def read(self, size=-1): return self[3].read(size) - + @property def stream(self): return self[3] - + #} END stream reader interface - - + + class ODeltaStream(OStream): """Uses size info of its stream, delaying reads""" - + def __new__(cls, sha, type, size, stream, *args, **kwargs): """Helps with the initialization of subclasses""" return tuple.__new__(cls, (sha, type, size, stream)) - + #{ Stream Reader Interface - + @property def size(self): return self[3].size - + #} END stream reader interface - - + + class OPackStream(OPackInfo): """Next to pack object information, a stream outputting an undeltified base object is provided""" __slots__ = tuple() - + def __new__(cls, packoffset, type, size, stream, *args): """Helps with the initialization of subclasses""" return tuple.__new__(cls, (packoffset, type, size, stream)) - - #{ Stream Reader Interface + + #{ Stream Reader Interface def read(self, size=-1): return self[3].read(size) - + @property def stream(self): return self[3] #} END stream reader interface - + class ODeltaPackStream(ODeltaPackInfo): """Provides a stream outputting the uncompressed offset delta information""" __slots__ = tuple() - + def __new__(cls, packoffset, type, size, delta_info, stream): return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) - #{ Stream Reader Interface + #{ Stream Reader Interface def read(self, size=-1): return self[4].read(size) - + @property def stream(self): return self[4] @@ -197,106 +197,106 @@ def stream(self): class IStream(list): - """Represents an input content stream to be fed into the ODB. It is mutable to allow + """Represents an input content stream to be fed into the ODB. It is mutable to allow the ODB to record information about the operations outcome right in this instance. - + It provides interfaces for the OStream and a StreamReader to allow the instance to blend in without prior conversion. - + The only method your content stream must support is 'read'""" __slots__ = tuple() - + def __new__(cls, type, size, stream, sha=None): return list.__new__(cls, (sha, type, size, stream, None)) - + def __init__(self, type, size, stream, sha=None): list.__init__(self, (sha, type, size, stream, None)) - - #{ Interface + + #{ Interface @property def hexsha(self): """:return: our sha, hex encoded, 40 bytes""" return bin_to_hex(self[0]) - + def _error(self): """:return: the error that occurred when processing the stream, or None""" return self[4] - + def _set_error(self, exc): """Set this input stream to the given exc, may be None to reset the error""" self[4] = exc - + error = property(_error, _set_error) - + #} END interface - + #{ Stream Reader Interface - + def read(self, size=-1): - """Implements a simple stream reader interface, passing the read call on + """Implements a simple stream reader interface, passing the read call on to our internal stream""" return self[3].read(size) - - #} END stream reader interface - + + #} END stream reader interface + #{ interface - + def _set_binsha(self, binsha): self[0] = binsha - + def _binsha(self): return self[0] - + binsha = property(_binsha, _set_binsha) - - + + def _type(self): return self[1] - + def _set_type(self, type): self[1] = type - + type = property(_type, _set_type) - + def _size(self): return self[2] - + def _set_size(self, size): self[2] = size - + size = property(_size, _set_size) - + def _stream(self): return self[3] - + def _set_stream(self, stream): self[3] = stream - + stream = property(_stream, _set_stream) - - #} END odb info interface - + + #} END odb info interface + class InvalidOInfo(tuple): - """Carries information about a sha identifying an object which is invalid in + """Carries information about a sha identifying an object which is invalid in the queried database. The exception attribute provides more information about the cause of the issue""" __slots__ = tuple() - + def __new__(cls, sha, exc): return tuple.__new__(cls, (sha, exc)) - + def __init__(self, sha, exc): tuple.__init__(self, (sha, exc)) - + @property def binsha(self): return self[0] - + @property def hexsha(self): return bin_to_hex(self[0]) - + @property def error(self): """:return: exception instance explaining the failure""" @@ -306,6 +306,5 @@ def error(self): class InvalidOStream(InvalidOInfo): """Carries information about an invalid ODB stream""" __slots__ = tuple() - -#} END ODB Bases +#} END ODB Bases diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 867e93a81..0eef1e5d5 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -4,20 +4,20 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains implementations of database retrieveing objects""" from gitdb.util import ( - pool, - join, - LazyMixin, - hex_to_bin - ) + pool, + join, + LazyMixin, + hex_to_bin +) from gitdb.exc import ( - BadObject, - AmbiguousObjectName - ) + BadObject, + AmbiguousObjectName +) from async import ( - ChannelThreadTask - ) + ChannelThreadTask +) from itertools import chain @@ -28,17 +28,17 @@ class ObjectDBR(object): """Defines an interface for object database lookup. Objects are identified either by their 20 byte bin sha""" - + def __contains__(self, sha): return self.has_obj - - #{ Query Interface + + #{ Query Interface def has_object(self, sha): """ :return: True if the object identified by the given 20 bytes binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") - + def has_object_async(self, reader): """Return a reader yielding information about the membership of objects as identified by shas @@ -46,62 +46,62 @@ def has_object_async(self, reader): :return: async.Reader yielding tuples of (sha, bool) pairs which indicate whether the given sha exists in the database or not""" task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) - return pool.add_task(task) - + return pool.add_task(task) + def info(self, sha): """ :return: OInfo instance :param sha: bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - + def info_async(self, reader): """Retrieve information of a multitude of objects asynchronously :param reader: Channel yielding the sha's of the objects of interest :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" task = ChannelThreadTask(reader, str(self.info_async), self.info) return pool.add_task(task) - + def stream(self, sha): """:return: OStream instance :param sha: 20 bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - + def stream_async(self, reader): """Retrieve the OStream of multiple objects :param reader: see ``info`` :param max_threads: see ``ObjectDBW.store`` :return: async.Reader yielding OStream|InvalidOStream instances in any order - - **Note:** depending on the system configuration, it might not be possible to + + **Note:** depending on the system configuration, it might not be possible to read all OStreams at once. Instead, read them individually using reader.read(x) where x is small enough.""" # base implementation just uses the stream method repeatedly task = ChannelThreadTask(reader, str(self.stream_async), self.stream) return pool.add_task(task) - + def size(self): """:return: amount of objects in this database""" raise NotImplementedError() - + def sha_iter(self): """Return iterator yielding 20 byte shas for all objects in this data base""" raise NotImplementedError() - + #} END query interface - - + + class ObjectDBW(object): """Defines an interface to create objects in the database""" - + def __init__(self, *args, **kwargs): self._ostream = None - + #{ Edit Interface def set_ostream(self, stream): """ Adjusts the stream to which all data should be sent when storing new objects - + :param stream: if not None, the stream to use, if None the default stream will be used. :return: previously installed stream, or None if there was no override @@ -109,96 +109,96 @@ def set_ostream(self, stream): cstream = self._ostream self._ostream = stream return cstream - + def ostream(self): """ :return: overridden output stream this instance will write to, or None if it will write to the default stream""" return self._ostream - + def store(self, istream): """ Create a new object in the database :return: the input istream object with its sha set to its corresponding value - - :param istream: IStream compatible instance. If its sha is already set - to a value, the object will just be stored in the our database format, + + :param istream: IStream compatible instance. If its sha is already set + to a value, the object will just be stored in the our database format, in which case the input stream is expected to be in object format ( header + contents ). :raise IOError: if data could not be written""" raise NotImplementedError("To be implemented in subclass") - + def store_async(self, reader): """ - Create multiple new objects in the database asynchronously. The method will - return right away, returning an output channel which receives the results as + Create multiple new objects in the database asynchronously. The method will + return right away, returning an output channel which receives the results as they are computed. - + :return: Channel yielding your IStream which served as input, in any order. - The IStreams sha will be set to the sha it received during the process, + The IStreams sha will be set to the sha it received during the process, or its error attribute will be set to the exception informing about the error. - + :param reader: async.Reader yielding IStream instances. The same instances will be used in the output channel as were received in by the Reader. - - **Note:** As some ODB implementations implement this operation atomic, they might - abort the whole operation if one item could not be processed. Hence check how + + **Note:** As some ODB implementations implement this operation atomic, they might + abort the whole operation if one item could not be processed. Hence check how many items have actually been produced.""" # base implementation uses store to perform the work - task = ChannelThreadTask(reader, str(self.store_async), self.store) + task = ChannelThreadTask(reader, str(self.store_async), self.store) return pool.add_task(task) - + #} END edit interface - + class FileDBBase(object): - """Provides basic facilities to retrieve files of interest, including + """Provides basic facilities to retrieve files of interest, including caching facilities to help mapping hexsha's to objects""" - + def __init__(self, root_path): """Initialize this instance to look for its files at the given root path All subsequent operations will be relative to this path - :raise InvalidDBRoot: + :raise InvalidDBRoot: **Note:** The base will not perform any accessablity checking as the base - might not yet be accessible, but become accessible before the first + might not yet be accessible, but become accessible before the first access.""" super(FileDBBase, self).__init__() self._root_path = root_path - - - #{ Interface + + + #{ Interface def root_path(self): """:return: path at which this db operates""" return self._root_path - + def db_path(self, rela_path): """ - :return: the given relative path relative to our database root, allowing + :return: the given relative path relative to our database root, allowing to pontentially access datafiles""" return join(self._root_path, rela_path) #} END interface - + class CachingDB(object): """A database which uses caches to speed-up access""" - - #{ Interface + + #{ Interface def update_cache(self, force=False): """ Call this method if the underlying data changed to trigger an update of the internal caching structures. - + :param force: if True, the update must be performed. Otherwise the implementation may decide not to perform an update if it thinks nothing has changed. :return: True if an update was performed as something change indeed""" - + # END interface def _databases_recursive(database, output): - """Fill output list with database from db, in order. Deals with Loose, Packed + """Fill output list with database from db, in order. Deals with Loose, Packed and compound databases.""" if isinstance(database, CompoundDB): compounds = list() @@ -209,11 +209,11 @@ def _databases_recursive(database, output): else: output.append(database) # END handle database type - + class CompoundDB(ObjectDBR, LazyMixin, CachingDB): """A database which delegates calls to sub-databases. - + Databases are stored in the lazy-loaded _dbs attribute. Define _set_cache_ to update it with your databases""" def _set_cache_(self, attr): @@ -223,27 +223,27 @@ def _set_cache_(self, attr): self._db_cache = dict() else: super(CompoundDB, self)._set_cache_(attr) - + def _db_query(self, sha): """:return: database containing the given 20 byte sha :raise BadObject:""" - # most databases use binary representations, prevent converting + # most databases use binary representations, prevent converting # it everytime a database is being queried try: return self._db_cache[sha] except KeyError: pass # END first level cache - + for db in self._dbs: if db.has_object(sha): self._db_cache[sha] = db return db # END for each database raise BadObject(sha) - - #{ ObjectDBR interface - + + #{ ObjectDBR interface + def has_object(self, sha): try: self._db_query(sha) @@ -251,24 +251,24 @@ def has_object(self, sha): except BadObject: return False # END handle exceptions - + def info(self, sha): return self._db_query(sha).info(sha) - + def stream(self, sha): return self._db_query(sha).stream(sha) def size(self): """:return: total size of all contained databases""" return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) - + def sha_iter(self): return chain(*(db.sha_iter() for db in self._dbs)) - + #} END object DBR Interface - + #{ Interface - + def databases(self): """:return: tuple of database instances we use for lookups""" return tuple(self._dbs) @@ -283,7 +283,7 @@ def update_cache(self, force=False): # END if is caching db # END for each database to update return stat - + def partial_to_complete_sha_hex(self, partial_hexsha): """ :return: 20 byte binary sha1 from the given less-than-40 byte hexsha @@ -291,14 +291,14 @@ def partial_to_complete_sha_hex(self, partial_hexsha): :raise AmbiguousObjectName: """ databases = list() _databases_recursive(self, databases) - + len_partial_hexsha = len(partial_hexsha) if len_partial_hexsha % 2 != 0: partial_binsha = hex_to_bin(partial_hexsha + "0") else: partial_binsha = hex_to_bin(partial_hexsha) - # END assure successful binary conversion - + # END assure successful binary conversion + candidate = None for db in databases: full_bin_sha = None @@ -320,7 +320,5 @@ def partial_to_complete_sha_hex(self, partial_hexsha): if not candidate: raise BadObject(partial_binsha) return candidate - - #} END interface - + #} END interface diff --git a/gitdb/db/git.py b/gitdb/db/git.py index 1d6ad0f26..6e6ec5d1f 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -14,10 +14,11 @@ from gitdb.util import LazyMixin from gitdb.exc import ( - InvalidDBRoot, - BadObject, - AmbiguousObjectName - ) + InvalidDBRoot, + BadObject, + AmbiguousObjectName +) + import os __all__ = ('GitDB', ) @@ -30,21 +31,21 @@ class GitDB(FileDBBase, ObjectDBW, CompoundDB): PackDBCls = PackedDB LooseDBCls = LooseObjectDB ReferenceDBCls = ReferenceDB - + # Directories packs_dir = 'pack' loose_dir = '' alternates_dir = os.path.join('info', 'alternates') - + def __init__(self, root_path): """Initialize ourselves on a git objects directory""" super(GitDB, self).__init__(root_path) - + def _set_cache_(self, attr): if attr == '_dbs' or attr == '_loose_db': self._dbs = list() loose_db = None - for subpath, dbcls in ((self.packs_dir, self.PackDBCls), + for subpath, dbcls in ((self.packs_dir, self.PackDBCls), (self.loose_dir, self.LooseDBCls), (self.alternates_dir, self.ReferenceDBCls)): path = self.db_path(subpath) @@ -55,31 +56,30 @@ def _set_cache_(self, attr): # END remember loose db # END check path exists # END for each db type - + # should have at least one subdb if not self._dbs: raise InvalidDBRoot(self.root_path()) # END handle error - + # we the first one should have the store method assert loose_db is not None and hasattr(loose_db, 'store'), "First database needs store functionality" - + # finally set the value self._loose_db = loose_db else: super(GitDB, self)._set_cache_(attr) # END handle attrs - + #{ ObjectDBW interface - + def store(self, istream): return self._loose_db.store(istream) - + def ostream(self): return self._loose_db.ostream() - + def set_ostream(self, ostream): return self._loose_db.set_ostream(ostream) - + #} END objectdbw interface - diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index dc0ea0e3b..4ebca84d3 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -10,46 +10,46 @@ from gitdb.exc import ( - InvalidDBRoot, + InvalidDBRoot, BadObject, AmbiguousObjectName - ) +) from gitdb.stream import ( DecompressMemMapReader, FDCompressedSha1Writer, FDStream, Sha1Writer - ) +) from gitdb.base import ( - OStream, - OInfo - ) + OStream, + OInfo +) from gitdb.util import ( - file_contents_ro_filepath, - ENOENT, - hex_to_bin, - bin_to_hex, - exists, - chmod, - isdir, - isfile, - remove, - mkdir, - rename, - dirname, - basename, - join - ) - -from gitdb.fun import ( + file_contents_ro_filepath, + ENOENT, + hex_to_bin, + bin_to_hex, + exists, + chmod, + isdir, + isfile, + remove, + mkdir, + rename, + dirname, + basename, + join +) + +from gitdb.fun import ( chunk_size, - loose_object_header_info, + loose_object_header_info, write_object, stream_copy - ) +) import tempfile import mmap @@ -62,11 +62,11 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): """A database which operates on loose object files""" - + # CONFIGURATION # chunks in which data will be copied between streams stream_chunk_size = chunk_size - + # On windows we need to keep it writable, otherwise it cannot be removed # either new_objects_mode = 0444 @@ -81,14 +81,14 @@ def __init__(self, root_path): # Depending on the root, this might work for some mounts, for others not, which # is why it is per instance self._fd_open_flags = getattr(os, 'O_NOATIME', 0) - - #{ Interface + + #{ Interface def object_path(self, hexsha): """ - :return: path at which the object with the given hexsha would be stored, + :return: path at which the object with the given hexsha would be stored, relative to the database root""" return join(hexsha[:2], hexsha[2:]) - + def readable_db_object_path(self, hexsha): """ :return: readable object path to the object identified by hexsha @@ -97,8 +97,8 @@ def readable_db_object_path(self, hexsha): return self._hexsha_to_file[hexsha] except KeyError: pass - # END ignore cache misses - + # END ignore cache misses + # try filesystem path = self.db_path(self.object_path(hexsha)) if exists(path): @@ -106,11 +106,11 @@ def readable_db_object_path(self, hexsha): return path # END handle cache raise BadObject(hexsha) - + def partial_to_complete_sha_hex(self, partial_hexsha): """:return: 20 byte binary sha1 string which matches the given name uniquely :param name: hexadecimal partial name - :raise AmbiguousObjectName: + :raise AmbiguousObjectName: :raise BadObject: """ candidate = None for binsha in self.sha_iter(): @@ -123,9 +123,9 @@ def partial_to_complete_sha_hex(self, partial_hexsha): if candidate is None: raise BadObject(partial_hexsha) return candidate - + #} END interface - + def _map_loose_object(self, sha): """ :return: memory map of that file to allow random read access @@ -151,13 +151,13 @@ def _map_loose_object(self, sha): finally: os.close(fd) # END assure file is closed - + def set_ostream(self, stream): """:raise TypeError: if the stream does not support the Sha1Writer interface""" if stream is not None and not isinstance(stream, Sha1Writer): raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) return super(LooseObjectDB, self).set_ostream(stream) - + def info(self, sha): m = self._map_loose_object(sha) try: @@ -166,12 +166,12 @@ def info(self, sha): finally: m.close() # END assure release of system resources - + def stream(self, sha): m = self._map_loose_object(sha) type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) return OStream(sha, type, size, stream) - + def has_object(self, sha): try: self.readable_db_object_path(bin_to_hex(sha)) @@ -179,7 +179,7 @@ def has_object(self, sha): except BadObject: return False # END check existance - + def store(self, istream): """note: The sha we produce will be hex by nature""" tmp_path = None @@ -187,14 +187,14 @@ def store(self, istream): if writer is None: # open a tmp file to write the data to fd, tmp_path = tempfile.mkstemp(prefix='obj', dir=self._root_path) - + if istream.binsha is None: writer = FDCompressedSha1Writer(fd) else: writer = FDStream(fd) # END handle direct stream copies # END handle custom writer - + try: try: if istream.binsha is not None: @@ -215,14 +215,14 @@ def store(self, istream): os.remove(tmp_path) raise # END assure tmpfile removal on error - + hexsha = None if istream.binsha: hexsha = istream.hexsha else: hexsha = writer.sha(as_hex=True) # END handle sha - + if tmp_path: obj_path = self.db_path(self.object_path(hexsha)) obj_dir = dirname(obj_path) @@ -234,29 +234,28 @@ def store(self, istream): remove(obj_path) # END handle win322 rename(tmp_path, obj_path) - + # make sure its readable for all ! It started out as rw-- tmp file # but needs to be rwrr chmod(obj_path, self.new_objects_mode) # END handle dry_run - + istream.binsha = hex_to_bin(hexsha) return istream - + def sha_iter(self): # find all files which look like an object, extract sha from there for root, dirs, files in os.walk(self.root_path()): root_base = basename(root) if len(root_base) != 2: continue - + for f in files: if len(f) != 38: continue yield hex_to_bin(root_base + f) # END for each file # END for each walk iteration - + def size(self): return len(tuple(self.sha_iter())) - diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index b9b2b8995..e4fba94b3 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -10,18 +10,14 @@ ) from gitdb.base import ( - OStream, - IStream, - ) + OStream, + IStream, +) from gitdb.exc import ( - BadObject, - UnsupportedOperation - ) -from gitdb.stream import ( - ZippedStoreShaWriter, - DecompressMemMapReader, - ) + BadObject, + UnsupportedOperation +) from cStringIO import StringIO @@ -32,45 +28,45 @@ class MemoryDB(ObjectDBR, ObjectDBW): retrieval. It should be used to buffer results and obtain SHAs before writing it to the actual physical storage, as it allows to query whether object already exists in the target storage before introducing actual IO - + **Note:** memory is currently not threadsafe, hence the async methods cannot be used for storing""" - + def __init__(self): super(MemoryDB, self).__init__() self._db = LooseObjectDB("path/doesnt/matter") - + # maps 20 byte shas to their OStream objects self._cache = dict() - + def set_ostream(self, stream): raise UnsupportedOperation("MemoryDB's always stream into memory") - + def store(self, istream): zstream = ZippedStoreShaWriter() self._db.set_ostream(zstream) - + istream = self._db.store(istream) zstream.close() # close to flush zstream.seek(0) - - # don't provide a size, the stream is written in object format, hence the + + # don't provide a size, the stream is written in object format, hence the # header needs decompression - decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) + decomp_stream = DecompressMemMapReader(zstream.getvalue(), close_on_deletion=False) self._cache[istream.binsha] = OStream(istream.binsha, istream.type, istream.size, decomp_stream) - + return istream - + def store_async(self, reader): raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") - + def has_object(self, sha): return sha in self._cache def info(self, sha): # we always return streams, which are infos as well return self.stream(sha) - + def stream(self, sha): try: ostream = self._cache[sha] @@ -80,15 +76,15 @@ def stream(self, sha): except KeyError: raise BadObject(sha) # END exception handling - + def size(self): return len(self._cache) - + def sha_iter(self): return self._cache.iterkeys() - - - #{ Interface + + + #{ Interface def stream_copy(self, sha_iter, odb): """Copy the streams as identified by sha's yielded by sha_iter into the given odb The streams will be copied directly @@ -100,12 +96,12 @@ def stream_copy(self, sha_iter, odb): if odb.has_object(sha): continue # END check object existance - + ostream = self.stream(sha) # compressed data including header sio = StringIO(ostream.stream.data()) istream = IStream(ostream.type, ostream.size, sio, sha) - + odb.store(istream) count += 1 # END for each sha diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 928731937..09f811847 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -12,10 +12,10 @@ from gitdb.util import LazyMixin from gitdb.exc import ( - BadObject, - UnsupportedOperation, - AmbiguousObjectName - ) + BadObject, + UnsupportedOperation, + AmbiguousObjectName +) from gitdb.pack import PackEntity @@ -29,12 +29,12 @@ class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): """A database operating on a set of object packs""" - + # sort the priority list every N queries - # Higher values are better, performance tests don't show this has + # Higher values are better, performance tests don't show this has # any effect, but it should have one _sort_interval = 500 - + def __init__(self, root_path): super(PackedDB, self).__init__(root_path) # list of lists with three items: @@ -44,29 +44,29 @@ def __init__(self, root_path): # self._entities = list() # lazy loaded list self._hit_count = 0 # amount of hits self._st_mtime = 0 # last modification data of our root path - + def _set_cache_(self, attr): if attr == '_entities': self._entities = list() self.update_cache(force=True) # END handle entities initialization - + def _sort_entities(self): self._entities.sort(key=lambda l: l[0], reverse=True) - + def _pack_info(self, sha): """:return: tuple(entity, index) for an item at the given sha :param sha: 20 or 40 byte sha :raise BadObject: **Note:** This method is not thread-safe, but may be hit in multi-threaded - operation. The worst thing that can happen though is a counter that + operation. The worst thing that can happen though is a counter that was not incremented, or the list being in wrong order. So we safe the time for locking here, lets see how that goes""" # presort ? if self._hit_count % self._sort_interval == 0: self._sort_entities() # END update sorting - + for item in self._entities: index = item[2](sha) if index is not None: @@ -75,14 +75,14 @@ def _pack_info(self, sha): return (item[1], index) # END index found in pack # END for each item - + # no hit, see whether we have to update packs # NOTE: considering packs don't change very often, we safe this call # and leave it to the super-caller to trigger that raise BadObject(sha) - - #{ Object DB Read - + + #{ Object DB Read + def has_object(self, sha): try: self._pack_info(sha) @@ -90,15 +90,15 @@ def has_object(self, sha): except BadObject: return False # END exception handling - + def info(self, sha): entity, index = self._pack_info(sha) return entity.info_at_index(index) - + def stream(self, sha): entity, index = self._pack_info(sha) return entity.stream_at_index(index) - + def sha_iter(self): sha_list = list() for entity in self.entities(): @@ -108,50 +108,50 @@ def sha_iter(self): yield sha_by_index(index) # END for each index # END for each entity - + def size(self): sizes = [item[1].index().size() for item in self._entities] return reduce(lambda x,y: x+y, sizes, 0) - + #} END object db read - + #{ object db write - + def store(self, istream): - """Storing individual objects is not feasible as a pack is designed to + """Storing individual objects is not feasible as a pack is designed to hold multiple objects. Writing or rewriting packs for single objects is inefficient""" raise UnsupportedOperation() - + def store_async(self, reader): # TODO: add ObjectDBRW before implementing this raise NotImplementedError() - + #} END object db write - - - #{ Interface - + + + #{ Interface + def update_cache(self, force=False): """ - Update our cache with the acutally existing packs on disk. Add new ones, + Update our cache with the acutally existing packs on disk. Add new ones, and remove deleted ones. We keep the unchanged ones - + :param force: If True, the cache will be updated even though the directory does not appear to have changed according to its modification timestamp. - :return: True if the packs have been updated so there is new information, + :return: True if the packs have been updated so there is new information, False if there was no change to the pack database""" stat = os.stat(self.root_path()) if not force and stat.st_mtime <= self._st_mtime: return False # END abort early on no change self._st_mtime = stat.st_mtime - + # packs are supposed to be prefixed with pack- by git-convention # get all pack files, figure out what changed pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) our_pack_files = set(item[1].pack().path() for item in self._entities) - + # new packs for pack_file in (pack_files - our_pack_files): # init the hit-counter/priority with the size, a good measure for hit- @@ -159,7 +159,7 @@ def update_cache(self, force=False): entity = PackEntity(pack_file) self._entities.append([entity.pack().size(), entity, entity.index().sha_to_index]) # END for each new packfile - + # removed packs for pack_file in (our_pack_files - pack_files): del_index = -1 @@ -172,22 +172,22 @@ def update_cache(self, force=False): assert del_index != -1 del(self._entities[del_index]) # END for each removed pack - + # reinitialize prioritiess self._sort_entities() return True - + def entities(self): """:return: list of pack entities operated upon by this database""" return [ item[1] for item in self._entities ] - + def partial_to_complete_sha(self, partial_binsha, canonical_length): """:return: 20 byte sha as inferred by the given partial binary sha - :param partial_binsha: binary sha with less than 20 bytes + :param partial_binsha: binary sha with less than 20 bytes :param canonical_length: length of the corresponding canonical representation. It is required as binary sha's cannot display whether the original hex sha had an odd or even number of characters - :raise AmbiguousObjectName: + :raise AmbiguousObjectName: :raise BadObject: """ candidate = None for item in self._entities: @@ -199,11 +199,11 @@ def partial_to_complete_sha(self, partial_binsha, canonical_length): candidate = sha # END handle full sha could be found # END for each entity - + if candidate: return candidate - + # still not found ? raise BadObject(partial_binsha) - + #} END interface diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 60004a77a..368ab9a61 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -11,16 +11,16 @@ class ReferenceDB(CompoundDB): """A database consisting of database referred to in a file""" - + # Configuration # Specifies the object database to use for the paths found in the alternates # file. If None, it defaults to the GitDB ObjectDBCls = None - + def __init__(self, ref_file): super(ReferenceDB, self).__init__() self._ref_file = ref_file - + def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() @@ -28,7 +28,7 @@ def _set_cache_(self, attr): else: super(ReferenceDB, self)._set_cache_(attr) # END handle attrs - + def _update_dbs_from_ref_file(self): dbcls = self.ObjectDBCls if dbcls is None: @@ -36,7 +36,7 @@ def _update_dbs_from_ref_file(self): from git import GitDB dbcls = GitDB # END get db type - + # try to get as many as possible, don't fail if some are unavailable ref_paths = list() try: @@ -44,10 +44,10 @@ def _update_dbs_from_ref_file(self): except (OSError, IOError): pass # END handle alternates - + ref_paths_set = set(ref_paths) cur_ref_paths_set = set(db.root_path() for db in self._dbs) - + # remove existing for path in (cur_ref_paths_set - ref_paths_set): for i, db in enumerate(self._dbs[:]): @@ -56,7 +56,7 @@ def _update_dbs_from_ref_file(self): continue # END del matching db # END for each path to remove - + # add new # sort them to maintain order added_paths = sorted(ref_paths_set - cur_ref_paths_set, key=lambda p: ref_paths.index(p)) @@ -72,7 +72,7 @@ def _update_dbs_from_ref_file(self): # ignore invalid paths or issues pass # END for each path to add - + def update_cache(self, force=False): # re-read alternates and update databases self._update_dbs_from_ref_file() diff --git a/gitdb/exc.py b/gitdb/exc.py index 7180fb586..47fc80912 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -7,17 +7,17 @@ class ODBError(Exception): """All errors thrown by the object database""" - + class InvalidDBRoot(ODBError): """Thrown if an object database cannot be initialized at the given path""" - + class BadObject(ODBError): - """The object with the given SHA does not exist. Instantiate with the + """The object with the given SHA does not exist. Instantiate with the failed sha""" - + def __str__(self): return "BadObject: %s" % to_hex_sha(self.args[0]) - + class ParseError(ODBError): """Thrown if the parsing of a file failed due to an invalid format""" diff --git a/gitdb/fun.py b/gitdb/fun.py index c1e73e895..9e5c44a6c 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -24,30 +24,30 @@ delta_types = (OFS_DELTA, REF_DELTA) type_id_to_type_map = { - 0 : "", # EXT 1 - 1 : "commit", - 2 : "tree", - 3 : "blob", - 4 : "tag", - 5 : "", # EXT 2 - OFS_DELTA : "OFS_DELTA", # OFFSET DELTA - REF_DELTA : "REF_DELTA" # REFERENCE DELTA - } + 0 : "", # EXT 1 + 1 : "commit", + 2 : "tree", + 3 : "blob", + 4 : "tag", + 5 : "", # EXT 2 + OFS_DELTA : "OFS_DELTA", # OFFSET DELTA + REF_DELTA : "REF_DELTA" # REFERENCE DELTA +} type_to_type_id_map = dict( - commit=1, - tree=2, - blob=3, - tag=4, - OFS_DELTA=OFS_DELTA, - REF_DELTA=REF_DELTA - ) + commit=1, + tree=2, + blob=3, + tag=4, + OFS_DELTA=OFS_DELTA, + REF_DELTA=REF_DELTA +) # used when dealing with larger streams -chunk_size = 1000*mmap.PAGESIZE +chunk_size = 1000 * mmap.PAGESIZE -__all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', - 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', +__all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') @@ -59,11 +59,11 @@ def _set_delta_rbound(d, size): to our size :return: d""" d.ts = size - + # NOTE: data is truncated automatically when applying the delta # MUST NOT DO THIS HERE return d - + def _move_delta_lbound(d, bytes): """Move the delta by the given amount of bytes, reducing its size so that its right bound stays static @@ -71,19 +71,19 @@ def _move_delta_lbound(d, bytes): :return: d""" if bytes == 0: return - + d.to += bytes d.so += bytes d.ts -= bytes if d.data is not None: d.data = d.data[bytes:] # END handle data - + return d - + def delta_duplicate(src): return DeltaChunk(src.to, src.ts, src.so, src.data) - + def delta_chunk_apply(dc, bbuf, write): """Apply own data to the target buffer :param bbuf: buffer providing source bytes for copy operations @@ -107,13 +107,13 @@ class DeltaChunk(object): """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" __slots__ = ( - 'to', # start offset in the target buffer in bytes + 'to', # start offset in the target buffer in bytes 'ts', # size of this chunk in the target buffer in bytes 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None ) - + def __init__(self, to, ts, so, data): self.to = to self.ts = ts @@ -122,22 +122,22 @@ def __init__(self, to, ts, so, data): def __repr__(self): return "DeltaChunk(%i, %i, %s, %s)" % (self.to, self.ts, self.so, self.data or "") - + #{ Interface - + def rbound(self): return self.to + self.ts - + def has_data(self): """:return: True if the instance has data to add to the target stream""" return self.data is not None - + #} END interface def _closest_index(dcl, absofs): """:return: index at which the given absofs should be inserted. The index points to the DeltaChunk with a target buffer absofs that equals or is greater than - absofs. + absofs. **Note:** global method for performance only, it belongs to DeltaChunkList""" lo = 0 hi = len(dcl) @@ -153,7 +153,7 @@ def _closest_index(dcl, absofs): # END handle bound # END for each delta absofs return len(dcl)-1 - + def delta_list_apply(dcl, bbuf, write): """Apply the chain's changes and write the final result using the passed write function. @@ -166,14 +166,14 @@ def delta_list_apply(dcl, bbuf, write): # END for each dc def delta_list_slice(dcl, absofs, size, ndcl): - """:return: Subsection of this list at the given absolute offset, with the given + """:return: Subsection of this list at the given absolute offset, with the given size in bytes. :return: None""" cdi = _closest_index(dcl, absofs) # delta start index cd = dcl[cdi] slen = len(dcl) - lappend = ndcl.append - + lappend = ndcl.append + if cd.to != absofs: tcd = DeltaChunk(cd.to, cd.ts, cd.so, cd.data) _move_delta_lbound(tcd, absofs - cd.to) @@ -182,7 +182,7 @@ def delta_list_slice(dcl, absofs, size, ndcl): size -= tcd.ts cdi += 1 # END lbound overlap handling - + while cdi < slen and size: # are we larger than the current block cd = dcl[cdi] @@ -198,38 +198,38 @@ def delta_list_slice(dcl, absofs, size, ndcl): # END hadle size cdi += 1 # END for each chunk - - + + class DeltaChunkList(list): """List with special functionality to deal with DeltaChunks. There are two types of lists we represent. The one was created bottom-up, working - towards the latest delta, the other kind was created top-down, working from the - latest delta down to the earliest ancestor. This attribute is queryable + towards the latest delta, the other kind was created top-down, working from the + latest delta down to the earliest ancestor. This attribute is queryable after all processing with is_reversed.""" - + __slots__ = tuple() - + def rbound(self): """:return: rightmost extend in bytes, absolute""" if len(self) == 0: return 0 return self[-1].rbound() - + def lbound(self): """:return: leftmost byte at which this chunklist starts""" if len(self) == 0: return 0 return self[0].to - + def size(self): """:return: size of bytes as measured by our delta chunks""" return self.rbound() - self.lbound() - + def apply(self, bbuf, write): """Only used by public clients, internally we only use the global routines for performance""" return delta_list_apply(self, bbuf, write) - + def compress(self): """Alter the list to reduce the amount of nodes. Currently we concatenate add-chunks @@ -239,7 +239,7 @@ def compress(self): return self i = 0 slen_orig = slen - + first_data_index = None while i < slen: dc = self[i] @@ -253,27 +253,27 @@ def compress(self): xdc = self[x] nd.write(xdc.data[:xdc.ts]) # END collect data - + del(self[first_data_index:i-1]) buf = nd.getvalue() - self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) - + self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) + slen = len(self) i = first_data_index + 1 - + # END concatenate data first_data_index = None continue # END skip non-data chunks - + if first_data_index is None: first_data_index = i-1 # END iterate list - + #if slen_orig != len(self): # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) return self - + def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches target_size @@ -283,39 +283,39 @@ def check_integrity(self, target_size=-1): assert self[-1].rbound() == target_size assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size # END target size verification - + if len(self) < 2: return - + # check data for dc in self: assert dc.ts > 0 if dc.has_data(): assert len(dc.data) >= dc.ts # END for each dc - + left = islice(self, 0, len(self)-1) right = iter(self) right.next() - # this is very pythonic - we might have just use index based access here, + # this is very pythonic - we might have just use index based access here, # but this could actually be faster for lft,rgt in izip(left, right): assert lft.rbound() == rgt.to assert lft.to + lft.ts == rgt.to # END for each pair - + class TopdownDeltaChunkList(DeltaChunkList): - """Represents a list which is generated by feeding its ancestor streams one by + """Represents a list which is generated by feeding its ancestor streams one by one""" - __slots__ = tuple() - + __slots__ = tuple() + def connect_with_next_base(self, bdcl): """Connect this chain with the next level of our base delta chunklist. The goal in this game is to mark as many of our chunks rigid, hence they - cannot be changed by any of the upcoming bases anymore. Once all our + cannot be changed by any of the upcoming bases anymore. Once all our chunks are marked like that, we can stop all processing - :param bdcl: data chunk list being one of our bases. They must be fed in + :param bdcl: data chunk list being one of our bases. They must be fed in consequtively and in order, towards the earliest ancestor delta :return: True if processing was done. Use it to abort processing of remaining streams if False is returned""" @@ -326,13 +326,13 @@ def connect_with_next_base(self, bdcl): while dci < slen: dc = self[dci] dci += 1 - + # all add-chunks which are already topmost don't need additional processing if dc.data is not None: nfc += 1 continue # END skip add chunks - + # copy chunks # integrate the portion of the base list into ourselves. Lists # dont support efficient insertion ( just one at a time ), but for now @@ -341,13 +341,13 @@ def connect_with_next_base(self, bdcl): # ourselves in order to reduce the amount of insertions ... del(ccl[:]) delta_list_slice(bdcl, dc.so, dc.ts, ccl) - + # move the target bounds into place to match with our chunk ofs = dc.to - dc.so for cdc in ccl: cdc.to += ofs # END update target bounds - + if len(ccl) == 1: self[dci-1] = ccl[0] else: @@ -359,19 +359,19 @@ def connect_with_next_base(self, bdcl): del(self[dci-1:]) # include deletion of dc self.extend(ccl) self.extend(post_dci) - + slen = len(self) dci += len(ccl)-1 # deleted dc, added rest - + # END handle chunk replacement # END for each chunk - + if nfc == slen: return False # END handle completeness return True - - + + #} END structures #{ Routines @@ -386,14 +386,14 @@ def is_loose_object(m): def loose_object_header_info(m): """ - :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the + :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) type_name, size = hdr[:hdr.find("\0")].split(" ") return type_name, int(size) - + def pack_object_header_info(data): """ :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) @@ -417,7 +417,7 @@ def create_pack_object_header(obj_type, obj_size): """ :return: string defining the pack header comprised of the object type and its incompressed size in bytes - + :param obj_type: pack type_id of the object :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte @@ -432,10 +432,10 @@ def create_pack_object_header(obj_type, obj_size): #END until size is consumed hdr += chr(c) return hdr - + def msb_size(data, offset=0): """ - :return: tuple(read_bytes, size) read the msb size from the given random + :return: tuple(read_bytes, size) read the msb size from the given random access data starting at the given byte offset""" size = 0 i = 0 @@ -452,19 +452,19 @@ def msb_size(data, offset=0): # END while in range if not hit_msb: raise AssertionError("Could not find terminating MSB byte in data stream") - return i+offset, size - + return i+offset, size + def loose_object_header(type, size): """ :return: string representing the loose object header, which is immediately followed by the content stream of size 'size'""" return "%s %i\0" % (type, size) - + def write_object(type, size, read, write, chunk_size=chunk_size): """ - Write the object as identified by type, size and source_stream into the + Write the object as identified by type, size and source_stream into the target_stream - + :param type: type string of the object :param size: amount of bytes to write from source_stream :param read: read method of a stream providing the content data @@ -473,26 +473,26 @@ def write_object(type, size, read, write, chunk_size=chunk_size): the routine exits, even if an error is thrown :return: The actual amount of bytes written to stream, which includes the header and a trailing newline""" tbw = 0 # total num bytes written - + # WRITE HEADER: type SP size NULL tbw += write(loose_object_header(type, size)) tbw += stream_copy(read, write, size, chunk_size) - + return tbw def stream_copy(read, write, size, chunk_size): """ - Copy a stream up to size bytes using the provided read and write methods, + Copy a stream up to size bytes using the provided read and write methods, in chunks of chunk_size - + **Note:** its much like stream_copy utility, but operates just using methods""" dbw = 0 # num data bytes written - + # WRITE ALL DATA UP TO SIZE while True: cs = min(chunk_size, size-dbw) # NOTE: not all write methods return the amount of written bytes, like - # mmap.write. Its bad, but we just deal with it ... perhaps its not + # mmap.write. Its bad, but we just deal with it ... perhaps its not # even less efficient # data_len = write(read(cs)) # dbw += data_len @@ -505,27 +505,27 @@ def stream_copy(read, write, size, chunk_size): # END check for stream end # END duplicate data return dbw - + def connect_deltas(dstreams): """ Read the condensed delta chunk information from dstream and merge its information into a list of existing delta chunks - + :param dstreams: iterable of delta stream objects, the delta to be applied last comes first, then all its ancestors in order :return: DeltaChunkList, containing all operations to apply""" tdcl = None # topmost dcl - + dcl = tdcl = TopdownDeltaChunkList() for dsi, ds in enumerate(dstreams): # print "Stream", dsi db = ds.read() delta_buf_size = ds.size - + # read header i, base_size = msb_size(db) i, target_size = msb_size(db, i) - + # interpret opcodes tbw = 0 # amount of target bytes written while i < delta_buf_size: @@ -554,15 +554,15 @@ def connect_deltas(dstreams): if (c & 0x40): cp_size |= (ord(db[i]) << 16) i += 1 - - if not cp_size: + + if not cp_size: cp_size = 0x10000 - + rbound = cp_off + cp_size if (rbound < cp_size or rbound > base_size): break - + dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) tbw += cp_size elif c: @@ -575,31 +575,31 @@ def connect_deltas(dstreams): raise ValueError("unexpected delta opcode 0") # END handle command byte # END while processing delta data - + dcl.compress() - + # merge the lists ! if dsi > 0: if not tdcl.connect_with_next_base(dcl): break # END handle merge - + # prepare next base dcl = DeltaChunkList() # END for each delta stream - + return tdcl - + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ Apply data from a delta buffer using a source buffer to the target file - + :param src_buf: random access data from which the delta was created :param src_buf_size: size of the source buffer in bytes :param delta_buf_size: size fo the delta buffer in bytes :param delta_buf: random access delta data :param write: write method taking a chunk of bytes - + **Note:** transcribed to python from the similar routine in patch-delta.c""" i = 0 db = delta_buf @@ -629,10 +629,10 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): if (c & 0x40): cp_size |= (ord(db[i]) << 16) i += 1 - - if not cp_size: + + if not cp_size: cp_size = 0x10000 - + rbound = cp_off + cp_size if (rbound < cp_size or rbound > src_buf_size): @@ -645,28 +645,28 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): raise ValueError("unexpected delta opcode 0") # END handle command byte # END while processing delta data - + # yes, lets use the exact same error message that git uses :) assert i == delta_buf_size, "delta replay has gone wild" - - + + def is_equal_canonical_sha(canonical_length, match, sha1): """ :return: True if the given lhs and rhs 20 byte binary shas - The comparison will take the canonical_length of the match sha into account, + The comparison will take the canonical_length of the match sha into account, hence the comparison will only use the last 4 bytes for uneven canonical representations :param match: less than 20 byte sha :param sha1: 20 byte sha""" binary_length = canonical_length/2 if match[:binary_length] != sha1[:binary_length]: return False - + if canonical_length - binary_length and \ (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: return False # END handle uneven canonnical length return True - + #} END routines diff --git a/gitdb/pack.py b/gitdb/pack.py index 48121f026..4a0badccb 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -54,9 +54,9 @@ ) from struct import ( - pack, - unpack, - ) + pack, + unpack, +) from binascii import crc32 @@ -68,10 +68,10 @@ __all__ = ('PackIndexFile', 'PackFile', 'PackEntity') - - -#{ Utilities + + +#{ Utilities def pack_object_at(cursor, offset, as_stream): """ @@ -81,13 +81,13 @@ def pack_object_at(cursor, offset, as_stream): data to be read decompressed. :param data: random accessable data containing all required information :parma offset: offset in to the data at which the object information is located - :param as_stream: if True, a stream object will be returned that can read + :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" data = cursor.use_region(offset).buffer() type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins delta_info = None - + # OFFSET DELTA if type_id == OFS_DELTA: i = data_rela_offset @@ -111,7 +111,7 @@ def pack_object_at(cursor, offset, as_stream): # assume its a base object total_rela_offset = data_rela_offset # END handle type id - + abs_data_offset = offset + total_rela_offset if as_stream: stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) @@ -141,29 +141,29 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): if want_crc: crc = base_crc #END initialize crc - + while True: chunk = read(chunk_size) br += len(chunk) compressed = zstream.compress(chunk) bw += len(compressed) write(compressed) # cannot assume return value - + if want_crc: crc = crc32(compressed, crc) #END handle crc - + if len(chunk) != chunk_size: break #END copy loop - + compressed = zstream.flush() bw += len(compressed) write(compressed) if want_crc: crc = crc32(compressed, crc) #END handle crc - + return (br, bw, crc) @@ -175,26 +175,26 @@ class IndexWriter(object): in one go to the given stream **Note:** currently only writes v2 indices""" __slots__ = '_objs' - + def __init__(self): self._objs = list() - + def append(self, binsha, crc, offset): """Append one piece of object information""" self._objs.append((binsha, crc, offset)) - + def write(self, pack_sha, write): """Write the index file using the given write method :param pack_sha: binary sha over the whole pack that we index :return: sha1 binary sha over all index file contents""" # sort for sha1 hash self._objs.sort(key=lambda o: o[0]) - + sha_writer = FlexibleSha1Writer(write) sha_write = sha_writer.write sha_write(PackIndexFile.index_v2_signature) sha_write(pack(">L", PackIndexFile.index_version_default)) - + # fanout tmplist = list((0,)*256) # fanout or list with 64 bit offsets for t in self._objs: @@ -206,16 +206,16 @@ def write(self, pack_sha, write): tmplist[i+1] += v #END write each fanout entry sha_write(pack('>L', tmplist[255])) - + # sha1 ordered # save calls, that is push them into c sha_write(''.join(t[0] for t in self._objs)) - + # crc32 for t in self._objs: sha_write(pack('>L', t[1]&0xffffffff)) #END for each crc - + tmplist = list() # offset 32 for t in self._objs: @@ -226,28 +226,28 @@ def write(self, pack_sha, write): #END hande 64 bit offsets sha_write(pack('>L', ofs&0xffffffff)) #END for each offset - + # offset 64 for ofs in tmplist: sha_write(pack(">Q", ofs)) #END for each offset - + # trailer assert(len(pack_sha) == 20) sha_write(pack_sha) sha = sha_writer.sha(as_hex=False) write(sha) return sha - - + + class PackIndexFile(LazyMixin): """A pack index provides offsets into the corresponding pack, allowing to find locations for offsets faster.""" - + # Dont use slots as we dynamically bind functions for each version, need a dict for this # The slots you see here are just to keep track of our instance variables - # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', + # __slots__ = ('_indexpath', '_fanout_table', '_cursor', '_version', # '_sha_list_offset', '_crc_list_offset', '_pack_offset', '_pack_64_offset') # used in v2 indices @@ -258,7 +258,7 @@ class PackIndexFile(LazyMixin): def __init__(self, indexpath): super(PackIndexFile, self).__init__() self._indexpath = indexpath - + def _set_cache_(self, attr): if attr == "_packfile_checksum": self._packfile_checksum = self._cursor.map()[-40:-20] @@ -276,91 +276,91 @@ def _set_cache_(self, attr): else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties - + # CHECK VERSION mmap = self._cursor.map() self._version = (mmap[:4] == self.index_v2_signature and 2) or 1 if self._version == 2: - version_id = unpack_from(">L", mmap, 4)[0] + version_id = unpack_from(">L", mmap, 4)[0] assert version_id == self._version, "Unsupported index version: %i" % version_id # END assert version - + # SETUP FUNCTIONS # setup our functions according to the actual version for fname in ('entry', 'offset', 'sha', 'crc'): setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) # END for each function to initialize - - + + # INITIALIZE DATA # byte offset is 8 if version is 2, 0 otherwise self._initialize() # END handle attributes - + #{ Access V1 - + def _entry_v1(self, i): """:return: tuple(offset, binsha, 0)""" - return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) - + return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) + def _offset_v1(self, i): """see ``_offset_v2``""" return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] - + def _sha_v1(self, i): """see ``_sha_v2``""" base = 1024 + (i*24)+4 return self._cursor.map()[base:base+20] - + def _crc_v1(self, i): """unsupported""" return 0 - + #} END access V1 - + #{ Access V2 def _entry_v2(self, i): """:return: tuple(offset, binsha, crc)""" return (self._offset_v2(i), self._sha_v2(i), self._crc_v2(i)) - + def _offset_v2(self, i): - """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only + """:return: 32 or 64 byte offset into pack files. 64 byte offsets will only be returned if the pack is larger than 4 GiB, or 2^32""" offset = unpack_from(">L", self._cursor.map(), self._pack_offset + i * 4)[0] - + # if the high-bit is set, this indicates that we have to lookup the offset # in the 64 bit region of the file. The current offset ( lower 31 bits ) # are the index into it if offset & 0x80000000: offset = unpack_from(">Q", self._cursor.map(), self._pack_64_offset + (offset & ~0x80000000) * 8)[0] # END handle 64 bit offset - + return offset - + def _sha_v2(self, i): """:return: sha at the given index of this file index instance""" base = self._sha_list_offset + i * 20 return self._cursor.map()[base:base+20] - + def _crc_v2(self, i): """:return: 4 bytes crc for the object at index i""" - return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] - + return unpack_from(">L", self._cursor.map(), self._crc_list_offset + i * 4)[0] + #} END access V2 - + #{ Initialization - + def _initialize(self): """initialize base data""" self._fanout_table = self._read_fanout((self._version == 2) * 8) - + if self._version == 2: self._crc_list_offset = self._sha_list_offset + self.size() * 20 self._pack_offset = self._crc_list_offset + self.size() * 4 self._pack_64_offset = self._pack_offset + self.size() * 4 # END setup base - + def _read_fanout(self, byte_offset): """Generate a fanout table from our data""" d = self._cursor.map() @@ -370,38 +370,38 @@ def _read_fanout(self, byte_offset): append(unpack_from('>L', d, byte_offset + i*4)[0]) # END for each entry return out - + #} END initialization - + #{ Properties def version(self): return self._version - + def size(self): """:return: amount of objects referred to by this index""" return self._fanout_table[255] - + def path(self): """:return: path to the packindexfile""" return self._indexpath - + def packfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of the pack file""" return self._cursor.map()[-40:-20] - + def indexfile_checksum(self): """:return: 20 byte sha representing the sha1 hash of this index file""" return self._cursor.map()[-20:] - + def offsets(self): """:return: sequence of all offsets in the order in which they were written - + **Note:** return value can be random accessed, but may be immmutable""" if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) - + # networkbyteorder to something array likes more if sys.byteorder == 'little': a.byteswap() @@ -409,7 +409,7 @@ def offsets(self): else: return tuple(self.offset(index) for index in xrange(self.size())) # END handle version - + def sha_to_index(self, sha): """ :return: index usable with the ``offset`` or ``entry`` method, or None @@ -421,7 +421,7 @@ def sha_to_index(self, sha): if first_byte != 0: lo = self._fanout_table[first_byte-1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - + # bisect until we have the sha while lo < hi: mid = (lo + hi) / 2 @@ -435,29 +435,29 @@ def sha_to_index(self, sha): # END handle midpoint # END bisect return None - + def partial_sha_to_index(self, partial_bin_sha, canonical_length): """ :return: index as in `sha_to_index` or None if the sha was not found in this index file :param partial_bin_sha: an at least two bytes of a partial binary sha - :param canonical_length: lenght of the original hexadecimal representation of the + :param canonical_length: lenght of the original hexadecimal representation of the given partial binary sha :raise AmbiguousObjectName:""" if len(partial_bin_sha) < 2: raise ValueError("Require at least 2 bytes of partial sha") - + first_byte = ord(partial_bin_sha[0]) get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: lo = self._fanout_table[first_byte-1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection - + # fill the partial to full 20 bytes filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) - - # find lowest + + # find lowest while lo < hi: mid = (lo + hi) / 2 c = cmp(filled_sha, get_sha(mid)) @@ -471,7 +471,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): lo = mid + 1 # END handle midpoint # END bisect - + if lo < self.size(): cur_sha = get_sha(lo) if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): @@ -484,86 +484,86 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): # END if we have a match # END if we found something return None - + if 'PackIndexFile_sha_to_index' in globals(): - # NOTE: Its just about 25% faster, the major bottleneck might be the attr + # NOTE: Its just about 25% faster, the major bottleneck might be the attr # accesses def sha_to_index(self, sha): return PackIndexFile_sha_to_index(self, sha) - # END redefine heavy-hitter with c version - + # END redefine heavy-hitter with c version + #} END properties - - + + class PackFile(LazyMixin): """A pack is a file written according to the Version 2 for git packs - + As we currently use memory maps, it could be assumed that the maximum size of - packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be + packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be fine though. - - **Note:** at some point, this might be implemented using streams as well, or + + **Note:** at some point, this might be implemented using streams as well, or streams are an alternate path in the case memory maps cannot be created - for some reason - one clearly doesn't want to read 10GB at once in that + for some reason - one clearly doesn't want to read 10GB at once in that case""" - + __slots__ = ('_packpath', '_cursor', '_size', '_version') pack_signature = 0x5041434b # 'PACK' pack_version_default = 2 - + # offset into our data at which the first object starts first_object_offset = 3*4 # header bytes footer_size = 20 # final sha - + def __init__(self, packpath): self._packpath = packpath - + def _set_cache_(self, attr): # we fill the whole cache, whichever attribute gets queried first self._cursor = mman.make_cursor(self._packpath).use_region() - + # read the header information type_id, self._version, self._size = unpack_from(">LLL", self._cursor.map(), 0) - + # TODO: figure out whether we should better keep the lock, or maybe # add a .keep file instead ? if type_id != self.pack_signature: raise ParseError("Invalid pack signature: %i" % type_id) - + def _iter_objects(self, start_offset, as_stream=True): """Handle the actual iteration of objects within this pack""" c = self._cursor content_size = c.file_size() - self.footer_size cur_offset = start_offset or self.first_object_offset - + null = NullStream() while cur_offset < content_size: data_offset, ostream = pack_object_at(c, cur_offset, True) # scrub the stream to the end - this decompresses the object, but yields # the amount of compressed bytes we need to get to the next offset - + stream_copy(ostream.read, null.write, ostream.size, chunk_size) cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() - - + + # if a stream is requested, reset it beforehand - # Otherwise return the Stream object directly, its derived from the + # Otherwise return the Stream object directly, its derived from the # info object if as_stream: ostream.stream.seek(0) yield ostream # END until we have read everything - + #{ Pack Information - + def size(self): - """:return: The amount of objects stored in this pack""" + """:return: The amount of objects stored in this pack""" return self._size - + def version(self): """:return: the version of this pack""" return self._version - + def data(self): """ :return: read-only data of this pack. It provides random access and usually @@ -571,22 +571,22 @@ def data(self): :note: This method is unsafe as it returns a window into a file which might be larger than than the actual window size""" # can use map as we are starting at offset 0. Otherwise we would have to use buffer() return self._cursor.use_region().map() - + def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] - + def path(self): """:return: path to the packfile""" return self._packpath #} END pack information - + #{ Pack Specific - + def collect_streams(self, offset): """ :return: list of pack streams which are required to build the object - at the given offset. The first entry of the list is the object at offset, + at the given offset. The first entry of the list is the object at offset, the last one is either a full object, or a REF_Delta stream. The latter type needs its reference object to be locked up in an ODB to form a valid delta chain. @@ -601,7 +601,7 @@ def collect_streams(self, offset): offset = ostream.pack_offset - ostream.delta_info else: # the only thing we can lookup are OFFSET deltas. Everything - # else is either an object, or a ref delta, in the latter + # else is either an object, or a ref delta, in the latter # case someone else has to find it break # END handle type @@ -609,55 +609,55 @@ def collect_streams(self, offset): return out #} END pack specific - + #{ Read-Database like Interface - + def info(self, offset): """Retrieve information about the object at the given file-absolute offset - + :param offset: byte offset :return: OPackInfo instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._cursor, offset or self.first_object_offset, False)[1] - + def stream(self, offset): """Retrieve an object at the given file-relative offset as stream along with its information - + :param offset: byte offset :return: OPackStream instance, the actual type differs depending on the type_id attribute""" return pack_object_at(self._cursor, offset or self.first_object_offset, True)[1] - + def stream_iter(self, start_offset=0): """ - :return: iterator yielding OPackStream compatible instances, allowing + :return: iterator yielding OPackStream compatible instances, allowing to access the data in the pack directly. - :param start_offset: offset to the first object to iterate. If 0, iteration + :param start_offset: offset to the first object to iterate. If 0, iteration starts at the very first object in the pack. - + **Note:** Iterating a pack directly is costly as the datastream has to be decompressed to determine the bounds between the objects""" return self._iter_objects(start_offset, as_stream=True) - + #} END Read-Database like Interface - - + + class PackEntity(LazyMixin): - """Combines the PackIndexFile and the PackFile into one, allowing the + """Combines the PackIndexFile and the PackFile into one, allowing the actual objects to be resolved and iterated""" - - __slots__ = ( '_index', # our index file + + __slots__ = ( '_index', # our index file '_pack', # our pack file '_offset_map' # on demand dict mapping one offset to the next consecutive one ) - + IndexFileCls = PackIndexFile PackFileCls = PackFile - + def __init__(self, pack_or_index_path): """Initialize ourselves with the path to the respective pack or index file""" basename, ext = os.path.splitext(pack_or_index_path) self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance - + def _set_cache_(self, attr): # currently this can only be _offset_map # TODO: make this a simple sorted offset array which can be bisected @@ -666,7 +666,7 @@ def _set_cache_(self, attr): offsets_sorted = sorted(self._index.offsets()) last_offset = len(self._pack.data()) - self._pack.footer_size assert offsets_sorted, "Cannot handle empty indices" - + offset_map = None if len(offsets_sorted) == 1: offset_map = { offsets_sorted[0] : last_offset } @@ -675,21 +675,21 @@ def _set_cache_(self, attr): iter_offsets_plus_one = iter(offsets_sorted) iter_offsets_plus_one.next() consecutive = izip(iter_offsets, iter_offsets_plus_one) - + offset_map = dict(consecutive) - + # the last offset is not yet set offset_map[offsets_sorted[-1]] = last_offset # END handle offset amount self._offset_map = offset_map - + def _sha_to_index(self, sha): """:return: index for the given sha, or raise""" index = self._index.sha_to_index(sha) if index is None: raise BadObject(sha) return index - + def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" _sha = self._index.sha @@ -697,7 +697,7 @@ def _iter_objects(self, as_stream): for index in xrange(self._index.size()): yield _object(_sha(index), as_stream, index) # END for each index - + def _object(self, sha, as_stream, index=-1): """:return: OInfo or OStream object providing information about the given sha :param index: if not -1, its assumed to be the sha's index in the IndexFile""" @@ -714,86 +714,86 @@ def _object(self, sha, as_stream, index=-1): packstream = self._pack.stream(offset) return OStream(sha, packstream.type, packstream.size, packstream.stream) # END handle non-deltas - + # produce a delta stream containing all info - # To prevent it from applying the deltas when querying the size, + # To prevent it from applying the deltas when querying the size, # we extract it from the delta stream ourselves streams = self.collect_streams_at_offset(offset) dstream = DeltaApplyReader.new(streams) - - return ODeltaStream(sha, dstream.type, None, dstream) + + return ODeltaStream(sha, dstream.type, None, dstream) else: if type_id not in delta_types: return OInfo(sha, type_id_to_type_map[type_id], uncomp_size) # END handle non-deltas - + # deltas are a little tougher - unpack the first bytes to obtain # the actual target size, as opposed to the size of the delta data streams = self.collect_streams_at_offset(offset) buf = streams[0].read(512) offset, src_size = msb_size(buf) offset, target_size = msb_size(buf, offset) - + # collect the streams to obtain the actual object type if streams[-1].type_id in delta_types: raise BadObject(sha, "Could not resolve delta object") - return OInfo(sha, streams[-1].type, target_size) + return OInfo(sha, streams[-1].type, target_size) # END handle stream - + #{ Read-Database like Interface - + def info(self, sha): """Retrieve information about the object identified by the given sha - + :param sha: 20 byte sha1 :raise BadObject: :return: OInfo instance, with 20 byte sha""" return self._object(sha, False) - + def stream(self, sha): """Retrieve an object stream along with its information as identified by the given sha - + :param sha: 20 byte sha1 - :raise BadObject: + :raise BadObject: :return: OStream instance, with 20 byte sha""" return self._object(sha, True) def info_at_index(self, index): """As ``info``, but uses a PackIndexFile compatible index to refer to the object""" return self._object(None, False, index) - + def stream_at_index(self, index): - """As ``stream``, but uses a PackIndexFile compatible index to refer to the + """As ``stream``, but uses a PackIndexFile compatible index to refer to the object""" return self._object(None, True, index) - + #} END Read-Database like Interface - - #{ Interface + + #{ Interface def pack(self): """:return: the underlying pack file instance""" return self._pack - + def index(self): """:return: the underlying pack index file instance""" return self._index - + def is_valid_stream(self, sha, use_crc=False): """ Verify that the stream at the given sha is valid. - - :param use_crc: if True, the index' crc is run over the compressed stream of + + :param use_crc: if True, the index' crc is run over the compressed stream of the object, which is much faster than checking the sha1. It is also more prone to unnoticed corruption or manipulation. :param sha: 20 byte sha1 of the object whose stream to verify - whether the compressed stream of the object is valid. If it is - a delta, this only verifies that the delta's data is valid, not the - data of the actual undeltified object, as it depends on more than + whether the compressed stream of the object is valid. If it is + a delta, this only verifies that the delta's data is valid, not the + data of the actual undeltified object, as it depends on more than just this stream. If False, the object will be decompressed and the sha generated. It must match the given sha - + :return: True if the stream is valid :raise UnsupportedOperation: If the index is version 1 only :raise BadObject: sha was not found""" @@ -801,12 +801,12 @@ def is_valid_stream(self, sha, use_crc=False): if self._index.version() < 2: raise UnsupportedOperation("Version 1 indices do not contain crc's, verify by sha instead") # END handle index version - + index = self._sha_to_index(sha) offset = self._index.offset(index) next_offset = self._offset_map[offset] crc_value = self._index.crc(index) - + # create the current crc value, on the compressed object data # Read it in chunks, without copying the data crc_update = zlib.crc32 @@ -819,7 +819,7 @@ def is_valid_stream(self, sha, use_crc=False): this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) cur_pos += size # END window size loop - + # crc returns signed 32 bit numbers, the AND op forces it into unsigned # mode ... wow, sneaky, from dulwich. return (this_crc_value & 0xffffffff) == crc_value @@ -828,7 +828,7 @@ def is_valid_stream(self, sha, use_crc=False): stream = self._object(sha, as_stream=True) # write a loose object, which is the basis for the sha write_object(stream.type, stream.size, stream.read, shawriter.write) - + assert shawriter.sha(as_hex=False) == sha return shawriter.sha(as_hex=False) == sha # END handle crc/sha verification @@ -839,21 +839,21 @@ def info_iter(self): :return: Iterator over all objects in this pack. The iterator yields OInfo instances""" return self._iter_objects(as_stream=False) - + def stream_iter(self): """ :return: iterator over all objects in this pack. The iterator yields OStream instances""" return self._iter_objects(as_stream=True) - + def collect_streams_at_offset(self, offset): """ As the version in the PackFile, but can resolve REF deltas within this pack For more info, see ``collect_streams`` - + :param offset: offset into the pack file at which the object can be found""" streams = self._pack.collect_streams(offset) - + # try to resolve the last one if needed. It is assumed to be either # a REF delta, or a base object, as OFFSET deltas are resolved by the pack if streams[-1].type_id == REF_DELTA: @@ -866,54 +866,54 @@ def collect_streams_at_offset(self, offset): stream = self._pack.stream(self._index.offset(sindex)) streams.append(stream) else: - # must be another OFS DELTA - this could happen if a REF - # delta we resolve previously points to an OFS delta. Who + # must be another OFS DELTA - this could happen if a REF + # delta we resolve previously points to an OFS delta. Who # would do that ;) ? We can handle it though stream = self._pack.stream(stream.delta_info) streams.append(stream) # END handle ref delta # END resolve ref streams # END resolve streams - + return streams - + def collect_streams(self, sha): """ As ``PackFile.collect_streams``, but takes a sha instead of an offset. Additionally, ref_delta streams will be resolved within this pack. If this is not possible, the stream will be left alone, hence it is adivsed - to check for unresolved ref-deltas and resolve them before attempting to + to check for unresolved ref-deltas and resolve them before attempting to construct a delta stream. - + :param sha: 20 byte sha1 specifying the object whose related streams you want to collect - :return: list of streams, first being the actual object delta, the last being + :return: list of streams, first being the actual object delta, the last being a possibly unresolved base object. :raise BadObject:""" return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) - - + + @classmethod - def write_pack(cls, object_iter, pack_write, index_write=None, + def write_pack(cls, object_iter, pack_write, index_write=None, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): """ Create a new pack by putting all objects obtained by the object_iterator into a pack which is written using the pack_write method. The respective index is produced as well if index_write is not Non. - + :param object_iter: iterator yielding odb output objects :param pack_write: function to receive strings to write into the pack stream :param indx_write: if not None, the function writes the index file corresponding to the pack. - :param object_count: if you can provide the amount of objects in your iteration, - this would be the place to put it. Otherwise we have to pre-iterate and store + :param object_count: if you can provide the amount of objects in your iteration, + this would be the place to put it. Otherwise we have to pre-iterate and store all items into a list to get the number, which uses more memory than necessary. :param zlib_compression: the zlib compression level to use :return: tuple(pack_sha, index_binsha) binary sha over all the contents of the pack and over all contents of the index. If index_write was None, index_binsha will be None - + **Note:** The destination of the write functions is up to the user. It could be a socket, or a file for instance - + **Note:** writes only undeltified objects""" objs = object_iter if not object_count: @@ -922,26 +922,26 @@ def write_pack(cls, object_iter, pack_write, index_write=None, #END handle list type object_count = len(objs) #END handle object - + pack_writer = FlexibleSha1Writer(pack_write) pwrite = pack_writer.write ofs = 0 # current offset into the pack file index = None wants_index = index_write is not None - + # write header pwrite(pack('>LLL', PackFile.pack_signature, PackFile.pack_version_default, object_count)) ofs += 12 - + if wants_index: index = IndexWriter() #END handle index header - + actual_count = 0 for obj in objs: actual_count += 1 crc = 0 - + # object header hdr = create_pack_object_header(obj.type_id, obj.size) if index_write: @@ -950,7 +950,7 @@ def write_pack(cls, object_iter, pack_write, index_write=None, crc = None #END handle crc pwrite(hdr) - + # data stream zstream = zlib.compressobj(zlib_compression) ostream = obj.stream @@ -959,54 +959,54 @@ def write_pack(cls, object_iter, pack_write, index_write=None, if wants_index: index.append(obj.binsha, crc, ofs) #END handle index - + ofs += len(hdr) + bw if actual_count == object_count: break #END abort once we are done #END for each object - + if actual_count != object_count: raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) #END count assertion - + # write footer pack_sha = pack_writer.sha(as_hex = False) assert len(pack_sha) == 20 pack_write(pack_sha) ofs += len(pack_sha) # just for completeness ;) - + index_sha = None if wants_index: index_sha = index.write(pack_sha, index_write) #END handle index - + return pack_sha, index_sha - + @classmethod def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): """Create a new on-disk entity comprised of a properly named pack file and a properly named and corresponding index file. The pack contains all OStream objects contained in object iter. :param base_dir: directory which is to contain the files :return: PackEntity instance initialized with the new pack - + **Note:** for more information on the other parameters see the write_pack method""" pack_fd, pack_path = tempfile.mkstemp('', 'pack', base_dir) index_fd, index_path = tempfile.mkstemp('', 'index', base_dir) pack_write = lambda d: os.write(pack_fd, d) index_write = lambda d: os.write(index_fd, d) - + pack_binsha, index_binsha = cls.write_pack(object_iter, pack_write, index_write, object_count, zlib_compression) os.close(pack_fd) os.close(index_fd) - + fmt = "pack-%s.%s" new_pack_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'pack')) new_index_path = os.path.join(base_dir, fmt % (bin_to_hex(pack_binsha), 'idx')) os.rename(pack_path, new_pack_path) os.rename(index_path, new_index_path) - + return cls(new_pack_path) - - + + #} END interface diff --git a/gitdb/stream.py b/gitdb/stream.py index 6441b1e1a..b21c39c9d 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -33,7 +33,7 @@ except ImportError: pass -__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', +__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', 'FDStream', 'NullStream') @@ -41,27 +41,27 @@ #{ RO Streams class DecompressMemMapReader(LazyMixin): - """Reads data in chunks from a memory map and decompresses it. The client sees + """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand buffered decompression accordingly - - A constraint on the total size of bytes is activated, simulating + + A constraint on the total size of bytes is activated, simulating a logical file within a possibly larger physical memory area - - To read efficiently, you clearly don't want to read individual bytes, instead, + + To read efficiently, you clearly don't want to read individual bytes, instead, read a few kilobytes at least. - - **Note:** The chunk-size should be carefully selected as it will involve quite a bit - of string copying due to the way the zlib is implemented. Its very wasteful, - hence we try to find a good tradeoff between allocation time and number of + + **Note:** The chunk-size should be carefully selected as it will involve quite a bit + of string copying due to the way the zlib is implemented. Its very wasteful, + hence we try to find a good tradeoff between allocation time and number of times we actually allocate. An own zlib implementation would be good here to better support streamed reading - it would only need to keep the mmap and decompress it into chunks, thats all ... """ - __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', + __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', '_cbr', '_phi') - + max_read_size = 512*1024 # currently unused - + def __init__(self, m, close_on_deletion, size=None): """Initialize with mmap for stream reading :param m: must be content data - use new if you have object data and no size""" @@ -77,22 +77,22 @@ def __init__(self, m, close_on_deletion, size=None): self._cbr = 0 # number of compressed bytes read self._phi = False # is True if we parsed the header info self._close = close_on_deletion # close the memmap on deletion ? - + def _set_cache_(self, attr): assert attr == '_s' - # only happens for size, which is a marker to indicate we still + # only happens for size, which is a marker to indicate we still # have to parse the header from the stream self._parse_header_info() - + def __del__(self): if self._close: self._m.close() # END handle resource freeing - + def _parse_header_info(self): - """If this stream contains object data, parse the header info and skip the + """If this stream contains object data, parse the header info and skip the stream to a point where each read will yield object content - + :return: parsed type_string, size""" # read header maxb = 512 # should really be enough, cgit uses 8192 I believe @@ -102,28 +102,28 @@ def _parse_header_info(self): type, size = hdr[:hdrend].split(" ") size = int(size) self._s = size - + # adjust internal state to match actual header length that we ignore # The buffer will be depleted first on future reads self._br = 0 hdrend += 1 # count terminating \0 self._buf = StringIO(hdr[hdrend:]) self._buflen = len(hdr) - hdrend - + self._phi = True - + return type, size - - #{ Interface - + + #{ Interface + @classmethod def new(self, m, close_on_deletion=False): """Create a new DecompressMemMapReader instance for acting as a read-only stream - This method parses the object header from m and returns the parsed + This method parses the object header from m and returns the parsed type and size, as well as the created stream instance. - + :param m: memory map on which to oparate. It must be object data ( header + contents ) - :param close_on_deletion: if True, the memory map will be closed once we are + :param close_on_deletion: if True, the memory map will be closed once we are being deleted""" inst = DecompressMemMapReader(m, close_on_deletion, 0) type, size = inst._parse_header_info() @@ -131,30 +131,30 @@ def new(self, m, close_on_deletion=False): def data(self): """:return: random access compatible data we are working on""" - return self._m - + return self._m + def compressed_bytes_read(self): """ - :return: number of compressed bytes read. This includes the bytes it + :return: number of compressed bytes read. This includes the bytes it took to decompress the header ( if there was one )""" # ABSTRACT: When decompressing a byte stream, it can be that the first - # x bytes which were requested match the first x bytes in the loosely + # x bytes which were requested match the first x bytes in the loosely # compressed datastream. This is the worst-case assumption that the reader # does, it assumes that it will get at least X bytes from X compressed bytes # in call cases. - # The caveat is that the object, according to our known uncompressed size, + # The caveat is that the object, according to our known uncompressed size, # is already complete, but there are still some bytes left in the compressed # stream that contribute to the amount of compressed bytes. # How can we know that we are truly done, and have read all bytes we need - # to read ? - # Without help, we cannot know, as we need to obtain the status of the + # to read ? + # Without help, we cannot know, as we need to obtain the status of the # decompression. If it is not finished, we need to decompress more data # until it is finished, to yield the actual number of compressed bytes # belonging to the decompressed object - # We are using a custom zlib module for this, if its not present, + # We are using a custom zlib module for this, if its not present, # we try to put in additional bytes up for decompression if feasible # and check for the unused_data. - + # Only scrub the stream forward if we are officially done with the # bytes we were to have. if self._br == self._s and not self._zip.unused_data: @@ -171,45 +171,45 @@ def compressed_bytes_read(self): self.read(mmap.PAGESIZE) # END scrub-loop default zlib # END handle stream scrubbing - + # reset bytes read, just to be sure self._br = self._s # END handle stream scrubbing - + # unused data ends up in the unconsumed tail, which was removed # from the count already return self._cbr - - #} END interface - + + #} END interface + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0""" if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset - + self._zip = zlib.decompressobj() self._br = self._cws = self._cwe = self._cbr = 0 if self._phi: self._phi = False del(self._s) # trigger header parsing on first access # END skip header - + def read(self, size=-1): if size < 1: size = self._s - self._br else: size = min(size, self._s - self._br) # END clamp size - + if size == 0: return str() # END handle depletion - - - # deplete the buffer, then just continue using the decompress object - # which has an own buffer. We just need this to transparently parse the + + + # deplete the buffer, then just continue using the decompress object + # which has an own buffer. We just need this to transparently parse the # header from the zlib stream dat = str() if self._buf: @@ -223,26 +223,26 @@ def read(self, size=-1): dat = self._buf.read() # ouch, duplicates data size -= self._buflen self._br += self._buflen - + self._buflen = 0 self._buf = None # END handle buffer len # END handle buffer - + # decompress some data - # Abstract: zlib needs to operate on chunks of our memory map ( which may + # Abstract: zlib needs to operate on chunks of our memory map ( which may # be large ), as it will otherwise and always fill in the 'unconsumed_tail' - # attribute which possible reads our whole map to the end, forcing + # attribute which possible reads our whole map to the end, forcing # everything to be read from disk even though just a portion was requested. - # As this would be a nogo, we workaround it by passing only chunks of data, - # moving the window into the memory map along as we decompress, which keeps + # As this would be a nogo, we workaround it by passing only chunks of data, + # moving the window into the memory map along as we decompress, which keeps # the tail smaller than our chunk-size. This causes 'only' the chunk to be # copied once, and another copy of a part of it when it creates the unconsumed # tail. We have to use it to hand in the appropriate amount of bytes durin g # the next read. tail = self._zip.unconsumed_tail if tail: - # move the window, make it as large as size demands. For code-clarity, + # move the window, make it as large as size demands. For code-clarity, # we just take the chunk from our map again instead of reusing the unconsumed # tail. The latter one would safe some memory copying, but we could end up # with not getting enough data uncompressed, so we had to sort that out as well. @@ -253,18 +253,18 @@ def read(self, size=-1): else: cws = self._cws self._cws = self._cwe - self._cwe = cws + size + self._cwe = cws + size # END handle tail - - + + # if window is too small, make it larger so zip can decompress something if self._cwe - self._cws < 8: self._cwe = self._cws + 8 # END adjust winsize - - # takes a slice, but doesn't copy the data, it says ... + + # takes a slice, but doesn't copy the data, it says ... indata = buffer(self._m, self._cws, self._cwe - self._cws) - + # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) dcompdat = self._zip.decompress(indata, size) @@ -274,13 +274,13 @@ def read(self, size=-1): # if we hit the end of the stream self._cbr += len(indata) - len(self._zip.unconsumed_tail) self._br += len(dcompdat) - + if dat: dcompdat = dat + dcompdat # END prepend our cached data - - # it can happen, depending on the compression, that we get less bytes - # than ordered as it needs the final portion of the data as well. + + # it can happen, depending on the compression, that we get less bytes + # than ordered as it needs the final portion of the data as well. # Recursively resolve that. # Note: dcompdat can be empty even though we still appear to have bytes # to read, if we are called by compressed_bytes_read - it manipulates @@ -290,30 +290,30 @@ def read(self, size=-1): # END handle special case return dcompdat - + class DeltaApplyReader(LazyMixin): - """A reader which dynamically applies pack deltas to a base object, keeping the + """A reader which dynamically applies pack deltas to a base object, keeping the memory demands to a minimum. - - The size of the final object is only obtainable once all deltas have been + + The size of the final object is only obtainable once all deltas have been applied, unless it is retrieved from a pack index. - + The uncompressed Delta has the following layout (MSB being a most significant bit encoded dynamic size): - + * MSB Source Size - the size of the base against which the delta was created * MSB Target Size - the size of the resulting data after the delta was applied * A list of one byte commands (cmd) which are followed by a specific protocol: - + * cmd & 0x80 - copy delta_data[offset:offset+size] - + * Followed by an encoded offset into the delta data * Followed by an encoded size of the chunk to copy - + * cmd & 0x7f - insert - + * insert cmd bytes from the delta buffer into the output stream - + * cmd == 0 - invalid operation ( or error in delta stream ) """ __slots__ = ( @@ -321,38 +321,38 @@ class DeltaApplyReader(LazyMixin): "_dstreams", # tuple of delta stream readers "_mm_target", # memory map of the delta-applied data "_size", # actual number of bytes in _mm_target - "_br" # number of bytes read + "_br" # number of bytes read ) - + #{ Configuration k_max_memory_move = 250*1000*1000 #} END configuration - + def __init__(self, stream_list): - """Initialize this instance with a list of streams, the first stream being + """Initialize this instance with a list of streams, the first stream being the delta to apply on top of all following deltas, the last stream being the base object onto which to apply the deltas""" assert len(stream_list) > 1, "Need at least one delta and one base stream" - + self._bstream = stream_list[-1] self._dstreams = tuple(stream_list[:-1]) self._br = 0 - + def _set_cache_too_slow_without_c(self, attr): - # the direct algorithm is fastest and most direct if there is only one + # the direct algorithm is fastest and most direct if there is only one # delta. Also, the extra overhead might not be worth it for items smaller - # than X - definitely the case in python, every function call costs + # than X - definitely the case in python, every function call costs # huge amounts of time # if len(self._dstreams) * self._bstream.size < self.k_max_memory_move: if len(self._dstreams) == 1: return self._set_cache_brute_(attr) - - # Aggregate all deltas into one delta in reverse order. Hence we take + + # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) dcl = connect_deltas(self._dstreams) - + # call len directly, as the (optional) c version doesn't implement the sequence # protocol if dcl.rbound() == 0: @@ -360,22 +360,22 @@ def _set_cache_too_slow_without_c(self, attr): self._mm_target = allocate_memory(0) return # END handle empty list - + self._size = dcl.rbound() self._mm_target = allocate_memory(self._size) - + bbuf = allocate_memory(self._bstream.size) stream_copy(self._bstream.read, bbuf.write, self._bstream.size, 256 * mmap.PAGESIZE) - + # APPLY CHUNKS write = self._mm_target.write dcl.apply(bbuf, write) - + self._mm_target.seek(0) - + def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" - + # TODO: There should be a special case if there is only one stream # Then the default-git algorithm should perform a tad faster, as the # delta is not peaked into, causing less overhead. @@ -388,37 +388,37 @@ def _set_cache_brute_(self, attr): buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) max_target_size = max(max_target_size, target_size) # END for each delta stream - + # sanity check - the first delta to apply should have the same source # size as our actual base stream base_size = self._bstream.size target_size = max_target_size - + # if we have more than 1 delta to apply, we will swap buffers, hence we must # assure that all buffers we use are large enough to hold all the results if len(self._dstreams) > 1: base_size = target_size = max(base_size, max_target_size) # END adjust buffer sizes - - + + # Allocate private memory map big enough to hold the first base buffer # We need random access to it bbuf = allocate_memory(base_size) stream_copy(self._bstream.read, bbuf.write, base_size, 256 * mmap.PAGESIZE) - + # allocate memory map large enough for the largest (intermediate) target - # We will use it as scratch space for all delta ops. If the final + # We will use it as scratch space for all delta ops. If the final # target buffer is smaller than our allocated space, we just use parts # of it upon return. tbuf = allocate_memory(target_size) - - # for each delta to apply, memory map the decompressed delta and + + # for each delta to apply, memory map the decompressed delta and # work on the op-codes to reconstruct everything. # For the actual copying, we use a seek and write pattern of buffer # slices. final_target_size = None for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): - # allocate a buffer to hold all delta data - fill in the data for + # allocate a buffer to hold all delta data - fill in the data for # fast access. We do this as we know that reading individual bytes # from our stream would be slower than necessary ( although possible ) # The dbuf buffer contains commands after the first two MSB sizes, the @@ -427,37 +427,37 @@ def _set_cache_brute_(self, attr): ddata.write(dbuf) # read the rest from the stream. The size we give is larger than necessary stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) - + ####################################################################### if 'c_apply_delta' in globals(): c_apply_delta(bbuf, ddata, tbuf); else: apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) ####################################################################### - - # finally, swap out source and target buffers. The target is now the + + # finally, swap out source and target buffers. The target is now the # base for the next delta to apply bbuf, tbuf = tbuf, bbuf bbuf.seek(0) tbuf.seek(0) final_target_size = target_size # END for each delta to apply - + # its already seeked to 0, constrain it to the actual size # NOTE: in the end of the loop, it swaps buffers, hence our target buffer # is not tbuf, but bbuf ! self._mm_target = bbuf self._size = final_target_size - - + + #{ Configuration if not has_perf_mod: _set_cache_ = _set_cache_brute_ else: _set_cache_ = _set_cache_too_slow_without_c - + #} END configuration - + def read(self, count=0): bl = self._size - self._br # bytes left if count < 1 or count > bl: @@ -467,63 +467,63 @@ def read(self, count=0): data = self._mm_target.read(count) self._br += len(data) return data - + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Allows to reset the stream to restart reading - + :raise ValueError: If offset and whence are not 0""" if offset != 0 or whence != getattr(os, 'SEEK_SET', 0): raise ValueError("Can only seek to position 0") # END handle offset self._br = 0 self._mm_target.seek(0) - - #{ Interface - + + #{ Interface + @classmethod def new(cls, stream_list): """ Convert the given list of streams into a stream which resolves deltas when reading from it. - + :param stream_list: two or more stream objects, first stream is a Delta to the object that you want to resolve, followed by N additional delta streams. The list's last stream must be a non-delta stream. - - :return: Non-Delta OPackStream object whose stream can be used to obtain + + :return: Non-Delta OPackStream object whose stream can be used to obtain the decompressed resolved data :raise ValueError: if the stream list cannot be handled""" if len(stream_list) < 2: raise ValueError("Need at least two streams") # END single object special handling - + if stream_list[-1].type_id in delta_types: raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) # END check stream - + return cls(stream_list) - + #} END interface - - + + #{ OInfo like Interface - + @property def type(self): return self._bstream.type - + @property def type_id(self): return self._bstream.type_id - + @property def size(self): """:return: number of uncompressed bytes in the stream""" return self._size - - #} END oinfo like interface - - + + #} END oinfo like interface + + #} END RO streams @@ -533,7 +533,7 @@ class Sha1Writer(object): """Simple stream writer which produces a sha whenever you like as it degests everything it is supposed to write""" __slots__ = "sha1" - + def __init__(self): self.sha1 = make_sha() @@ -545,29 +545,29 @@ def write(self, data): self.sha1.update(data) return len(data) - # END stream interface + # END stream interface #{ Interface - + def sha(self, as_hex = False): """:return: sha so far :param as_hex: if True, sha will be hex-encoded, binary otherwise""" if as_hex: return self.sha1.hexdigest() return self.sha1.digest() - - #} END interface + + #} END interface class FlexibleSha1Writer(Sha1Writer): - """Writer producing a sha1 while passing on the written bytes to the given + """Writer producing a sha1 while passing on the written bytes to the given write function""" __slots__ = 'writer' - + def __init__(self, writer): Sha1Writer.__init__(self) self.writer = writer - + def write(self, data): Sha1Writer.write(self, data) self.writer(data) @@ -580,18 +580,18 @@ def __init__(self): Sha1Writer.__init__(self) self.buf = StringIO() self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) - + def __getattr__(self, attr): return getattr(self.buf, attr) - + def write(self, data): alen = Sha1Writer.write(self, data) self.buf.write(self.zip.compress(data)) return alen - + def close(self): self.buf.write(self.zip.flush()) - + def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): """Seeking currently only supports to rewind written data Multiple writes are not supported""" @@ -599,23 +599,23 @@ def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)): raise ValueError("Can only seek to position 0") # END handle offset self.buf.seek(0) - + def getvalue(self): """:return: string value from the current stream position to the end""" return self.buf.getvalue() class FDCompressedSha1Writer(Sha1Writer): - """Digests data written to it, making the sha available, then compress the + """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor - + **Note:** operates on raw file descriptors **Note:** for this to work, you have to use the close-method of this instance""" __slots__ = ("fd", "sha1", "zip") - + # default exception exc = IOError("Failed to write all bytes to filedescriptor") - + def __init__(self, fd): super(FDCompressedSha1Writer, self).__init__() self.fd = fd @@ -643,33 +643,33 @@ def close(self): class FDStream(object): - """A simple wrapper providing the most basic functions on a file descriptor + """A simple wrapper providing the most basic functions on a file descriptor with the fileobject interface. Cannot use os.fdopen as the resulting stream takes ownership""" __slots__ = ("_fd", '_pos') def __init__(self, fd): self._fd = fd self._pos = 0 - + def write(self, data): self._pos += len(data) os.write(self._fd, data) - + def read(self, count=0): if count == 0: count = os.path.getsize(self._filepath) # END handle read everything - + bytes = os.read(self._fd, count) self._pos += len(bytes) return bytes - + def fileno(self): return self._fd - + def tell(self): return self._pos - + def close(self): close(self._fd) @@ -678,17 +678,15 @@ class NullStream(object): """A stream that does nothing but providing a stream interface. Use it like /dev/null""" __slots__ = tuple() - + def read(self, size=0): return '' - + def close(self): pass - + def write(self, data): return len(data) #} END W streams - - diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index f8059447f..e84e503b6 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -5,7 +5,7 @@ import gitdb.util -#{ Initialization +#{ Initialization def _init_pool(): """Assure the pool is actually threaded""" size = 2 diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 62614ee5c..dc89039dd 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -9,16 +9,16 @@ ZippedStoreShaWriter, fixture_path, TestBase - ) +) from gitdb.stream import Sha1Writer from gitdb.base import ( - IStream, - OStream, - OInfo - ) - + IStream, + OStream, + OInfo +) + from gitdb.exc import BadObject from gitdb.typ import str_blob_type @@ -28,14 +28,14 @@ __all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') - + class TestDBBase(TestBase): """Base class providing testing routines on databases""" - + # data two_lines = "1234\nhello world" all_data = (two_lines, ) - + def _assert_object_writing_simple(self, db): # write a bunch of objects and query their streams and info null_objs = db.size() @@ -46,23 +46,23 @@ def _assert_object_writing_simple(self, db): new_istream = db.store(istream) assert new_istream is istream assert db.has_object(istream.binsha) - + info = db.info(istream.binsha) assert isinstance(info, OInfo) assert info.type == istream.type and info.size == istream.size - + stream = db.stream(istream.binsha) assert isinstance(stream, OStream) assert stream.binsha == info.binsha and stream.type == info.type assert stream.read() == data # END for each item - + assert db.size() == null_objs + ni shas = list(db.sha_iter()) assert len(shas) == db.size() assert len(shas[0]) == 20 - - + + def _assert_object_writing(self, db): """General tests to verify object writing, compatible to ObjectDBW **Note:** requires write access to the database""" @@ -76,25 +76,25 @@ def _assert_object_writing(self, db): ostream = ostreamcls() assert isinstance(ostream, Sha1Writer) # END create ostream - + prev_ostream = db.set_ostream(ostream) - assert type(prev_ostream) in ostreams or prev_ostream in ostreams - + assert type(prev_ostream) in ostreams or prev_ostream in ostreams + istream = IStream(str_blob_type, len(data), StringIO(data)) - + # store returns same istream instance, with new sha set my_istream = db.store(istream) sha = istream.binsha assert my_istream is istream assert db.has_object(sha) != dry_run - assert len(sha) == 20 - + assert len(sha) == 20 + # verify data - the slow way, we want to run code if not dry_run: info = db.info(sha) assert str_blob_type == info.type assert info.size == len(data) - + ostream = db.stream(sha) assert ostream.read() == data assert ostream.type == str_blob_type @@ -102,29 +102,29 @@ def _assert_object_writing(self, db): else: self.failUnlessRaises(BadObject, db.info, sha) self.failUnlessRaises(BadObject, db.stream, sha) - + # DIRECT STREAM COPY # our data hase been written in object format to the StringIO # we pasesd as output stream. No physical database representation # was created. - # Test direct stream copy of object streams, the result must be + # Test direct stream copy of object streams, the result must be # identical to what we fed in ostream.seek(0) istream.stream = ostream assert istream.binsha is not None prev_sha = istream.binsha - + db.set_ostream(ZippedStoreShaWriter()) db.store(istream) assert istream.binsha == prev_sha new_ostream = db.ostream() - + # note: only works as long our store write uses the same compression # level, which is zip_best assert ostream.getvalue() == new_ostream.getvalue() # END for each data set # END for each dry_run mode - + def _assert_object_writing_async(self, db): """Test generic object writing using asynchronous access""" ni = 5000 @@ -134,23 +134,23 @@ def istream_generator(offset=0, ni=ni): yield IStream(str_blob_type, len(data), StringIO(data)) # END for each item # END generator utility - + # for now, we are very trusty here as we expect it to work if it worked # in the single-stream case - + # write objects reader = IteratorReader(istream_generator()) istream_reader = db.store_async(reader) istreams = istream_reader.read() # read all assert istream_reader.task().error() is None assert len(istreams) == ni - + for stream in istreams: assert stream.error is None assert len(stream.binsha) == 20 assert isinstance(stream, IStream) # END assert each stream - + # test has-object-async - we must have all previously added ones reader = IteratorReader( istream.binsha for istream in istreams ) hasobject_reader = db.has_object_async(reader) @@ -160,11 +160,11 @@ def istream_generator(offset=0, ni=ni): count += 1 # END for each sha assert count == ni - + # read the objects we have just written reader = IteratorReader( istream.binsha for istream in istreams ) ostream_reader = db.stream_async(reader) - + # read items individually to prevent hitting possible sys-limits count = 0 for ostream in ostream_reader: @@ -173,30 +173,30 @@ def istream_generator(offset=0, ni=ni): # END for each ostream assert ostream_reader.task().error() is None assert count == ni - + # get info about our items reader = IteratorReader( istream.binsha for istream in istreams ) info_reader = db.info_async(reader) - + count = 0 for oinfo in info_reader: assert isinstance(oinfo, OInfo) count += 1 # END for each oinfo instance assert count == ni - - + + # combined read-write using a converter # add 2500 items, and obtain their output streams nni = 2500 reader = IteratorReader(istream_generator(offset=ni, ni=nni)) istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] - + istream_reader = db.store_async(reader) istream_reader.set_post_cb(istream_to_sha) - + ostream_reader = db.stream_async(istream_reader) - + count = 0 # read it individually, otherwise we might run into the ulimit for ostream in ostream_reader: @@ -204,5 +204,3 @@ def istream_generator(offset=0, ni=ni): count += 1 # END for each ostream assert count == nni - - diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 1ef577aa3..4894c6a79 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -7,15 +7,15 @@ from gitdb.db import GitDB from gitdb.base import OStream, OInfo from gitdb.util import hex_to_bin, bin_to_hex - + class TestGitDB(TestDBBase): - + def test_reading(self): gdb = GitDB(fixture_path('../../../.git/objects')) - + # we have packs and loose objects, alternates doesn't necessarily exist assert 1 < len(gdb.databases()) < 4 - + # access should be possible gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert isinstance(gdb.info(gitdb_sha), OInfo) @@ -23,25 +23,25 @@ def test_reading(self): assert gdb.size() > 200 sha_list = list(gdb.sha_iter()) assert len(sha_list) == gdb.size() - - - # This is actually a test for compound functionality, but it doesn't + + + # This is actually a test for compound functionality, but it doesn't # have a separate test module # test partial shas # this one as uneven and quite short assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") - + # mix even/uneven hexshas for i, binsha in enumerate(sha_list): assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha # END for each sha - + self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") - + @with_rw_directory def test_writing(self, path): gdb = GitDB(path) - + # its possible to write objects self._assert_object_writing(gdb) self._assert_object_writing_async(gdb) diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index d7e1d01b0..e295db563 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -6,29 +6,28 @@ from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex - + class TestLooseDB(TestDBBase): - + @with_rw_directory def test_basics(self, path): ldb = LooseObjectDB(path) - + # write data self._assert_object_writing(ldb) self._assert_object_writing_async(ldb) - + # verify sha iteration and size shas = list(ldb.sha_iter()) assert shas and len(shas[0]) == 20 - + assert len(shas) == ldb.size() - + # verify find short object long_sha = bin_to_hex(shas[-1]) for short_sha in (long_sha[:20], long_sha[:5]): assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha # END for each sha - + self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') # raises if no object could be foudn - diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index df428e2b7..ac9bc34e9 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -4,27 +4,27 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from lib import * from gitdb.db import ( - MemoryDB, - LooseObjectDB - ) - + MemoryDB, + LooseObjectDB +) + class TestMemoryDB(TestDBBase): - + @with_rw_directory def test_writing(self, path): mdb = MemoryDB() - + # write data self._assert_object_writing_simple(mdb) - + # test stream copy ldb = LooseObjectDB(path) assert ldb.size() == 0 num_streams_copied = mdb.stream_copy(mdb.sha_iter(), ldb) assert num_streams_copied == mdb.size() - + assert ldb.size() == mdb.size() for sha in mdb.sha_iter(): assert ldb.has_object(sha) - assert ldb.stream(sha).read() == mdb.stream(sha).read() + assert ldb.stream(sha).read() == mdb.stream(sha).read() # END verify objects where copied and are equal diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index f4cb5bbc6..0d9110a01 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -12,45 +12,45 @@ import random class TestPackDB(TestDBBase): - + @with_rw_directory @with_packs_rw def test_writing(self, path): pdb = PackedDB(path) - + # on demand, we init our pack cache num_packs = len(pdb.entities()) assert pdb._st_mtime != 0 - - # test pack directory changed: + + # test pack directory changed: # packs removed - rename a file, should affect the glob pack_path = pdb.entities()[0].pack().path() new_pack_path = pack_path + "renamed" os.rename(pack_path, new_pack_path) - + pdb.update_cache(force=True) assert len(pdb.entities()) == num_packs - 1 - + # packs added os.rename(new_pack_path, pack_path) pdb.update_cache(force=True) assert len(pdb.entities()) == num_packs - + # bang on the cache # access the Entities directly, as there is no iteration interface # yet ( or required for now ) sha_list = list(pdb.sha_iter()) assert len(sha_list) == pdb.size() - + # hit all packs in random order random.shuffle(sha_list) - + for sha in sha_list: info = pdb.info(sha) stream = pdb.stream(sha) # END for each sha to query - - + + # test short finding - be a bit more brutal here max_bytes = 19 min_bytes = 2 @@ -64,10 +64,10 @@ def test_writing(self, path): pass # valid, we can have short objects # END exception handling # END for each sha to find - + # we should have at least one ambiguous, considering the small sizes - # but in our pack, there is no ambigious ... + # but in our pack, there is no ambigious ... # assert num_ambiguous - + # non-existing self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 1637bff74..086446823 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -6,14 +6,14 @@ from gitdb.db import ReferenceDB from gitdb.util import ( - NULL_BIN_SHA, - hex_to_bin - ) + NULL_BIN_SHA, + hex_to_bin +) import os - + class TestReferenceDB(TestDBBase): - + def make_alt_file(self, alt_path, alt_list): """Create an alternates file which contains the given alternates. The list can be empty""" @@ -21,40 +21,38 @@ def make_alt_file(self, alt_path, alt_list): for alt in alt_list: alt_file.write(alt + "\n") alt_file.close() - + @with_rw_directory def test_writing(self, path): NULL_BIN_SHA = '\0' * 20 - + alt_path = os.path.join(path, 'alternates') rdb = ReferenceDB(alt_path) assert len(rdb.databases()) == 0 assert rdb.size() == 0 assert len(list(rdb.sha_iter())) == 0 - + # try empty, non-existing assert not rdb.has_object(NULL_BIN_SHA) - - + + # setup alternate file # add two, one is invalid own_repo_path = fixture_path('../../../.git/objects') # use own repo self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) rdb.update_cache() assert len(rdb.databases()) == 1 - + # we should now find a default revision of ours gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert rdb.has_object(gitdb_sha) - + # remove valid self.make_alt_file(alt_path, ["just/one/invalid/path"]) rdb.update_cache() assert len(rdb.databases()) == 0 - + # add valid self.make_alt_file(alt_path, [own_repo_path]) rdb.update_cache() assert len(rdb.databases()) == 1 - - diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index ac8473a4e..685af2fad 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -4,12 +4,12 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" from gitdb import ( - OStream, + OStream, ) -from gitdb.stream import ( - Sha1Writer, - ZippedStoreShaWriter - ) +from gitdb.stream import ( + Sha1Writer, + ZippedStoreShaWriter +) from gitdb.util import zlib @@ -30,14 +30,14 @@ class TestBase(unittest.TestCase): """Base class for all tests""" - + #} END bases #{ Decorators def with_rw_directory(func): - """Create a temporary directory which can be written to, remove it if the + """Create a temporary directory which can be written to, remove it if the test suceeds, but leave it otherwise to aid additional debugging""" def wrapper(self): path = tempfile.mktemp(prefix=func.__name__) @@ -52,7 +52,7 @@ def wrapper(self): raise finally: # Need to collect here to be sure all handles have been closed. It appears - # a windows-only issue. In fact things should be deleted, as well as + # a windows-only issue. In fact things should be deleted, as well as # memory maps closed, once objects go out of scope. For some reason # though this is not the case here unless we collect explicitly. if not keep: @@ -60,20 +60,20 @@ def wrapper(self): shutil.rmtree(path) # END handle exception # END wrapper - + wrapper.__name__ = func.__name__ return wrapper def with_packs_rw(func): - """Function that provides a path into which the packs for testing should be + """Function that provides a path into which the packs for testing should be copied. Will pass on the path to the actual function afterwards""" def wrapper(self, path): src_pack_glob = fixture_path('packs/*') copy_files_globbed(src_pack_glob, path, hard_link_ok=True) return func(self, path) # END wrapper - + wrapper.__name__ = func.__name__ return wrapper @@ -86,10 +86,10 @@ def fixture_path(relapath=''): :param relapath: relative path into the fixtures directory, or '' to obtain the fixture directory itself""" return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) - + def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): """Copy all files found according to the given source glob into the target directory - :param hard_link_ok: if True, hard links will be created if possible. Otherwise + :param hard_link_ok: if True, hard links will be created if possible. Otherwise the files will be copied""" for src_file in glob.glob(source_glob): if hard_link_ok and hasattr(os, 'link'): @@ -103,7 +103,7 @@ def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): shutil.copy(src_file, target_dir) # END try hard link # END for each file to copy - + def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes @@ -121,7 +121,7 @@ def make_object(type, data): """:return: bytes resembling an uncompressed object""" odata = "blob %i\0" % len(data) return odata + data - + def make_memory_file(size_in_bytes, randomize=False): """:return: tuple(size_of_stream, stream) :param randomize: try to produce a very random stream""" @@ -137,14 +137,14 @@ def __init__(self): self.was_read = False self.bytes = 0 self.closed = False - + def read(self, size): self.was_read = True self.bytes = size - + def close(self): self.closed = True - + def _assert(self): assert self.was_read @@ -153,10 +153,9 @@ class DeriveTest(OStream): def __init__(self, sha, type, size, stream, *args, **kwargs): self.myarg = kwargs.pop('myarg') self.args = args - + def _assert(self): assert self.args assert self.myarg #} END stream utilitiess - diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index d4ce428c3..76b4d2709 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -6,7 +6,7 @@ from lib import ( TestBase, DummyStream, - DeriveTest, + DeriveTest, ) from gitdb import * @@ -20,33 +20,33 @@ class TestBaseTypes(TestBase): - + def test_streams(self): # test info sha = NULL_BIN_SHA s = 20 blob_id = 3 - + info = OInfo(sha, str_blob_type, s) assert info.binsha == sha assert info.type == str_blob_type assert info.type_id == blob_id assert info.size == s - + # test pack info # provides type_id pinfo = OPackInfo(0, blob_id, s) assert pinfo.type == str_blob_type assert pinfo.type_id == blob_id assert pinfo.pack_offset == 0 - + dpinfo = ODeltaPackInfo(0, blob_id, s, sha) assert dpinfo.type == str_blob_type assert dpinfo.type_id == blob_id assert dpinfo.delta_info == sha assert dpinfo.pack_offset == 0 - - + + # test ostream stream = DummyStream() ostream = OStream(*(info + (stream, ))) @@ -56,33 +56,33 @@ def test_streams(self): assert stream.bytes == 15 ostream.read(20) assert stream.bytes == 20 - + # test packstream postream = OPackStream(*(pinfo + (stream, ))) assert postream.stream is stream postream.read(10) stream._assert() assert stream.bytes == 10 - + # test deltapackstream dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) dpostream.stream is stream dpostream.read(5) stream._assert() assert stream.bytes == 5 - + # derive with own args DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() - + # test istream istream = IStream(str_blob_type, s, stream) assert istream.binsha == None istream.binsha = sha assert istream.binsha == sha - + assert len(istream.binsha) == 20 assert len(istream.hexsha) == 40 - + assert istream.size == s istream.size = s * 2 istream.size == s * 2 @@ -92,7 +92,7 @@ def test_streams(self): assert istream.stream is stream istream.stream = None assert istream.stream is None - + assert istream.error is None istream.error = Exception() assert isinstance(istream.error, Exception) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 611ae4299..f45063b1e 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -11,17 +11,17 @@ from cStringIO import StringIO from async import IteratorReader - + class TestExamples(TestBase): - + def test_base(self): ldb = LooseObjectDB(fixture_path("../../../.git/objects")) - + for sha1 in ldb.sha_iter(): oinfo = ldb.info(sha1) ostream = ldb.stream(sha1) assert oinfo[:3] == ostream[:3] - + assert len(ostream.read()) == ostream.size assert ldb.has_object(oinfo.binsha) # END for each sha in database @@ -32,33 +32,33 @@ def test_base(self): except UnboundLocalError: pass # END ignore exception if there are no loose objects - + data = "my data" istream = IStream("blob", len(data), StringIO(data)) - + # the object does not yet have a sha assert istream.binsha is None ldb.store(istream) # now the sha is set assert len(istream.binsha) == 20 assert ldb.has_object(istream.binsha) - - + + # async operation # Create a reader from an iterator reader = IteratorReader(ldb.sha_iter()) - + # get reader for object streams info_reader = ldb.stream_async(reader) - + # read one info = info_reader.read(1)[0] - + # read all the rest until depletion ostreams = info_reader.read() - + # set the pool to use two threads pool.set_size(2) - + # synchronize the mode of operation pool.set_size(0) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 779155a2a..f28aef4d4 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -12,15 +12,15 @@ from gitdb.stream import DeltaApplyReader from gitdb.pack import ( - PackEntity, - PackIndexFile, - PackFile - ) + PackEntity, + PackIndexFile, + PackFile +) from gitdb.base import ( - OInfo, - OStream, - ) + OInfo, + OStream, +) from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation @@ -39,15 +39,15 @@ def bin_sha_from_filename(filename): #} END utilities class TestPack(TestBase): - + packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) packindexfile_v2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx'), 2, 30) packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) - - + + def _assert_index_file(self, index, version, size): assert index.packfile_checksum() != index.indexfile_checksum() assert len(index.packfile_checksum()) == 20 @@ -55,93 +55,93 @@ def _assert_index_file(self, index, version, size): assert index.version() == version assert index.size() == size assert len(index.offsets()) == size - + # get all data of all objects for oidx in xrange(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) - + entry = index.entry(oidx) assert len(entry) == 3 - + assert entry[0] == index.offset(oidx) assert entry[1] == sha assert entry[2] == index.crc(oidx) - + # verify partial sha for l in (4,8,11,17,20): assert index.partial_sha_to_index(sha[:l], l*2) == oidx - + # END for each object index in indexfile self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) - - + + def _assert_pack_file(self, pack, version, size): assert pack.version() == 2 assert pack.size() == size assert len(pack.checksum()) == 20 - + num_obj = 0 for obj in pack.stream_iter(): num_obj += 1 info = pack.info(obj.pack_offset) stream = pack.stream(obj.pack_offset) - + assert info.pack_offset == stream.pack_offset assert info.type_id == stream.type_id assert hasattr(stream, 'read') - + # it should be possible to read from both streams assert obj.read() == stream.read() - + streams = pack.collect_streams(obj.pack_offset) assert streams - + # read the stream try: dstream = DeltaApplyReader.new(streams) except ValueError: - # ignore these, old git versions use only ref deltas, + # ignore these, old git versions use only ref deltas, # which we havent resolved ( as we are without an index ) # Also ignore non-delta streams continue # END get deltastream - + # read all data = dstream.read() assert len(data) == dstream.size - + # test seek dstream.seek(0) assert dstream.read() == data - - + + # read chunks # NOTE: the current implementation is safe, it basically transfers # all calls to the underlying memory map - + # END for each object assert num_obj == size - - + + def test_pack_index(self): # check version 1 and 2 - for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): + for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): index = PackIndexFile(indexfile) self._assert_index_file(index, version, size) # END run tests - + def test_pack(self): - # there is this special version 3, but apparently its like 2 ... + # there is this special version 3, but apparently its like 2 ... for packfile, version, size in (self.packfile_v2_3_ascii, self.packfile_v2_1, self.packfile_v2_2): pack = PackFile(packfile) self._assert_pack_file(pack, version, size) # END for each pack to test - + @with_rw_directory def test_pack_entity(self, rw_dir): pack_objs = list() - for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), + for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), (self.packfile_v2_2, self.packindexfile_v2), (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): packfile, version, size = packinfo @@ -150,7 +150,7 @@ def test_pack_entity(self, rw_dir): assert entity.pack().path() == packfile assert entity.index().path() == indexfile pack_objs.extend(entity.stream_iter()) - + count = 0 for info, stream in izip(entity.info_iter(), entity.stream_iter()): count += 1 @@ -158,10 +158,10 @@ def test_pack_entity(self, rw_dir): assert len(info.binsha) == 20 assert info.type_id == stream.type_id assert info.size == stream.size - + # we return fully resolved items, which is implied by the sha centric access assert not info.type_id in delta_types - + # try all calls assert len(entity.collect_streams(info.binsha)) oinfo = entity.info(info.binsha) @@ -170,7 +170,7 @@ def test_pack_entity(self, rw_dir): ostream = entity.stream(info.binsha) assert isinstance(ostream, OStream) assert ostream.binsha is not None - + # verify the stream try: assert entity.is_valid_stream(info.binsha, use_crc=True) @@ -180,16 +180,16 @@ def test_pack_entity(self, rw_dir): assert entity.is_valid_stream(info.binsha, use_crc=False) # END for each info, stream tuple assert count == size - + # END for each entity - + # pack writing - write all packs into one # index path can be None pack_path = tempfile.mktemp('', "pack", rw_dir) index_path = tempfile.mktemp('', 'index', rw_dir) iteration = 0 def rewind_streams(): - for obj in pack_objs: + for obj in pack_objs: obj.stream.seek(0) #END utility for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): @@ -199,23 +199,23 @@ def rewind_streams(): ifile = open(ipath, 'wb') iwrite = ifile.write #END handle ip - + # make sure we rewind the streams ... we work on the same objects over and over again - if iteration > 0: + if iteration > 0: rewind_streams() #END rewind streams iteration += 1 - + pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) pfile.close() assert os.path.getsize(ppath) > 100 - + # verify pack pf = PackFile(ppath) assert pf.size() == len(pack_objs) assert pf.version() == PackFile.pack_version_default assert pf.checksum() == pack_sha - + # verify index if ipath is not None: ifile.close() @@ -227,7 +227,7 @@ def rewind_streams(): assert idx.size() == len(pack_objs) #END verify files exist #END for each packpath, indexpath pair - + # verify the packs throughly rewind_streams() entity = PackEntity.create(pack_objs, rw_dir) @@ -239,9 +239,9 @@ def rewind_streams(): # END for each crc mode #END for each info assert count == len(pack_objs) - - + + def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets - # of course without really needing such a huge pack + # of course without really needing such a huge pack raise SkipTest() diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 6dc27463c..8360ea36c 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -32,18 +32,18 @@ class TestStream(TestBase): """Test stream classes""" - + data_sizes = (15, 10000, 1000*1024+512) - + def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): - """Make stream tests - the orig_stream is seekable, allowing it to be + """Make stream tests - the orig_stream is seekable, allowing it to be rewound and reused :param cdata: the data we expect to read from stream, the contents :param rewind_stream: function called to rewind the stream to make it ready for reuse""" ns = 10 assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) - + # read in small steps ss = len(cdata) / ns for i in range(ns): @@ -55,30 +55,30 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): if rest: assert rest == cdata[-len(rest):] # END handle rest - + if isinstance(stream, DecompressMemMapReader): assert len(stream.data()) == stream.compressed_bytes_read() # END handle special type - + rewind_stream(stream) - + # read everything rdata = stream.read() assert rdata == cdata - + if isinstance(stream, DecompressMemMapReader): assert len(stream.data()) == stream.compressed_bytes_read() # END handle special type - + def test_decompress_reader(self): for close_on_deletion in range(2): for with_size in range(2): for ds in self.data_sizes: cdata = make_bytes(ds, randomize=False) - + # zdata = zipped actual data # cdata = original content data - + # create reader if with_size: # need object data @@ -86,7 +86,7 @@ def test_decompress_reader(self): type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) assert size == len(cdata) assert type == str_blob_type - + # even if we don't set the size, it will be set automatically on first read test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) assert test_reader._s == len(cdata) @@ -95,60 +95,59 @@ def test_decompress_reader(self): zdata = zlib.compress(cdata) reader = DecompressMemMapReader(zdata, close_on_deletion, len(cdata)) assert reader._s == len(cdata) - # END get reader - + # END get reader + self._assert_stream_reader(reader, cdata, lambda r: r.seek(0)) - + # put in a dummy stream for closing dummy = DummyStream() reader._m = dummy - + assert not dummy.closed del(reader) assert dummy.closed == close_on_deletion # END for each datasize # END whether size should be used # END whether stream should be closed when deleted - + def test_sha_writer(self): writer = Sha1Writer() assert 2 == writer.write("hi") assert len(writer.sha(as_hex=1)) == 40 assert len(writer.sha(as_hex=0)) == 20 - + # make sure it does something ;) prev_sha = writer.sha() writer.write("hi again") assert writer.sha() != prev_sha - + def test_compressed_writer(self): for ds in self.data_sizes: fd, path = tempfile.mkstemp() ostream = FDCompressedSha1Writer(fd) data = make_bytes(ds, randomize=False) - + # for now, just a single write, code doesn't care about chunking assert len(data) == ostream.write(data) ostream.close() - + # its closed already self.failUnlessRaises(OSError, os.close, fd) - + # read everything back, compare to data we zip fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) written_data = os.read(fd, os.path.getsize(path)) assert len(written_data) == os.path.getsize(path) os.close(fd) assert written_data == zlib.compress(data, 1) # best speed - + os.remove(path) # END for each os - + def test_decompress_reader_special_case(self): odb = LooseObjectDB(fixture_path('objects')) ostream = odb.stream(hex_to_bin('7bb839852ed5e3a069966281bb08d50012fb309b')) - + # if there is a bug, we will be missing one byte exactly ! data = ostream.read() assert len(data) == ostream.size - diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 35f9f44a7..ed69f0d1f 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -8,28 +8,28 @@ from lib import TestBase from gitdb.util import ( - to_hex_sha, - to_bin_sha, - NULL_HEX_SHA, + to_hex_sha, + to_bin_sha, + NULL_HEX_SHA, LockedFD - ) +) + - class TestUtils(TestBase): def test_basics(self): assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA assert len(to_bin_sha(NULL_HEX_SHA)) == 20 assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA - + def _cmp_contents(self, file_path, data): - # raise if data from file at file_path + # raise if data from file at file_path # does not match data string fp = open(file_path, "rb") try: assert fp.read() == data finally: fp.close() - + def test_lockedfd(self): my_file = tempfile.mktemp() orig_data = "hello" @@ -37,43 +37,43 @@ def test_lockedfd(self): my_file_fp = open(my_file, "wb") my_file_fp.write(orig_data) my_file_fp.close() - + try: lfd = LockedFD(my_file) - lockfilepath = lfd._lockfilepath() - + lockfilepath = lfd._lockfilepath() + # cannot end before it was started self.failUnlessRaises(AssertionError, lfd.rollback) self.failUnlessRaises(AssertionError, lfd.commit) - + # open for writing assert not os.path.isfile(lockfilepath) wfd = lfd.open(write=True) assert lfd._fd is wfd assert os.path.isfile(lockfilepath) - + # write data and fail os.write(wfd, new_data) lfd.rollback() assert lfd._fd is None self._cmp_contents(my_file, orig_data) assert not os.path.isfile(lockfilepath) - + # additional call doesnt fail lfd.commit() lfd.rollback() - + # test reading lfd = LockedFD(my_file) rfd = lfd.open(write=False) assert os.read(rfd, len(orig_data)) == orig_data - + assert os.path.isfile(lockfilepath) # deletion rolls back del(lfd) assert not os.path.isfile(lockfilepath) - - + + # write data - concurrently lfd = LockedFD(my_file) olfd = LockedFD(my_file) @@ -82,17 +82,17 @@ def test_lockedfd(self): assert os.path.isfile(lockfilepath) # another one fails self.failUnlessRaises(IOError, olfd.open) - + wfdstream.write(new_data) lfd.commit() assert not os.path.isfile(lockfilepath) self._cmp_contents(my_file, new_data) - + # could test automatic _end_writing on destruction finally: os.remove(my_file) # END final cleanup - + # try non-existing file for reading lfd = LockedFD(tempfile.mktemp()) try: @@ -102,4 +102,3 @@ def test_lockedfd(self): else: self.fail("expected OSError") # END handle exceptions - diff --git a/gitdb/util.py b/gitdb/util.py index 1662b662d..b167b4d12 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -24,10 +24,9 @@ from async import ThreadPool from smmap import ( - StaticWindowMapManager, - SlidingWindowMapManager, - SlidingWindowMapBuffer - ) + StaticWindowMapManager, SlidingWindowMapManager, + SlidingWindowMapBuffer +) # initialize our global memory manager instance # Use it to free cached (and unused) resources. @@ -60,7 +59,7 @@ def unpack_from(fmt, data, offset=0): #{ Globals -# A pool distributing tasks, initially with zero threads, hence everything +# A pool distributing tasks, initially with zero threads, hence everything # will be handled in the main thread pool = ThreadPool(0) @@ -97,35 +96,35 @@ def unpack_from(fmt, data, offset=0): #} END Aliases -#{ compatibility stuff ... +#{ compatibility stuff ... class _RandomAccessStringIO(object): - """Wrapper to provide required functionality in case memory maps cannot or may + """Wrapper to provide required functionality in case memory maps cannot or may not be used. This is only really required in python 2.4""" __slots__ = '_sio' - + def __init__(self, buf=''): self._sio = StringIO(buf) - + def __getattr__(self, attr): return getattr(self._sio, attr) - + def __len__(self): return len(self.getvalue()) - + def __getitem__(self, i): return self.getvalue()[i] - + def __getslice__(self, start, end): return self.getvalue()[start:end] - + #} END compatibility stuff ... #{ Routines def make_sha(source=''): """A python2.4 workaround for the sha/hashlib module fiasco - + **Note** From the dulwich project """ try: return hashlib.sha1(source) @@ -138,25 +137,25 @@ def allocate_memory(size): if size == 0: return _RandomAccessStringIO('') # END handle empty chunks gracefully - + try: return mmap.mmap(-1, size) # read-write by default except EnvironmentError: # setup real memory instead # this of course may fail if the amount of memory is not available in - # one chunk - would only be the case in python 2.4, being more likely on + # one chunk - would only be the case in python 2.4, being more likely on # 32 bit systems. return _RandomAccessStringIO("\0"*size) # END handle memory allocation - + def file_contents_ro(fd, stream=False, allow_mmap=True): """:return: read-only contents of the file represented by the file descriptor fd - + :param fd: file descriptor opened for reading :param stream: if False, random access is provided, otherwise the stream interface is provided. - :param allow_mmap: if True, its allowed to map the contents into memory, which + :param allow_mmap: if True, its allowed to map the contents into memory, which allows large files to be handled and accessed efficiently. The file-descriptor will change its position if this is False""" try: @@ -171,24 +170,24 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): except OSError: pass # END exception handling - + # read manully contents = os.read(fd, os.fstat(fd).st_size) if stream: return _RandomAccessStringIO(contents) return contents - + def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): """Get the file contents at filepath as fast as possible - + :return: random access compatible memory of the given filepath :param stream: see ``file_contents_ro`` :param allow_mmap: see ``file_contents_ro`` :param flags: additional flags to pass to os.open :raise OSError: If the file could not be opened - - **Note** for now we don't try to use O_NOATIME directly as the right value needs to be - shared per database in fact. It only makes a real difference for loose object + + **Note** for now we don't try to use O_NOATIME directly as the right value needs to be + shared per database in fact. It only makes a real difference for loose object databases anyway, and they use it with the help of the ``flags`` parameter""" fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) try: @@ -196,19 +195,19 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): finally: close(fd) # END assure file is closed - + def sliding_ro_buffer(filepath, flags=0): """ :return: a buffer compatible object which uses our mapped memory manager internally ready to read the whole given filepath""" return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) - + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: return sha return bin_to_hex(sha) - + def to_bin_sha(sha): if len(sha) == 20: return sha @@ -227,12 +226,12 @@ class LazyMixin(object): is actually accessed and retrieved the first time. All future accesses will return the cached value as stored in the Instance's dict or slot. """ - + __slots__ = tuple() - + def __getattr__(self, attr): """ - Whenever an attribute is requested that we do not know, we allow it + Whenever an attribute is requested that we do not know, we allow it to be created and set. Next time the same attribute is reqeusted, it is simply returned from our dict/slots. """ self._set_cache_(attr) @@ -241,65 +240,65 @@ def __getattr__(self, attr): def _set_cache_(self, attr): """ - This method should be overridden in the derived class. + This method should be overridden in the derived class. It should check whether the attribute named by attr can be created and cached. Do nothing if you do not know the attribute or call your subclass - - The derived class may create as many additional attributes as it deems - necessary in case a git command returns more information than represented + + The derived class may create as many additional attributes as it deems + necessary in case a git command returns more information than represented in the single attribute.""" pass - + class LockedFD(object): """ This class facilitates a safe read and write operation to a file on disk. - If we write to 'file', we obtain a lock file at 'file.lock' and write to - that instead. If we succeed, the lock file will be renamed to overwrite + If we write to 'file', we obtain a lock file at 'file.lock' and write to + that instead. If we succeed, the lock file will be renamed to overwrite the original file. - - When reading, we obtain a lock file, but to prevent other writers from + + When reading, we obtain a lock file, but to prevent other writers from succeeding while we are reading the file. - - This type handles error correctly in that it will assure a consistent state + + This type handles error correctly in that it will assure a consistent state on destruction. - + **note** with this setup, parallel reading is not possible""" __slots__ = ("_filepath", '_fd', '_write') - + def __init__(self, filepath): """Initialize an instance with the givne filepath""" self._filepath = filepath self._fd = None self._write = None # if True, we write a file - + def __del__(self): # will do nothing if the file descriptor is already closed if self._fd is not None: self.rollback() - + def _lockfilepath(self): return "%s.lock" % self._filepath - + def open(self, write=False, stream=False): """ Open the file descriptor for reading or writing, both in binary mode. - + :param write: if True, the file descriptor will be opened for writing. Other wise it will be opened read-only. - :param stream: if True, the file descriptor will be wrapped into a simple stream + :param stream: if True, the file descriptor will be wrapped into a simple stream object which supports only reading or writing :return: fd to read from or write to. It is still maintained by this instance and must not be closed directly :raise IOError: if the lock could not be retrieved :raise OSError: If the actual file could not be opened for reading - + **note** must only be called once""" if self._write is not None: raise AssertionError("Called %s multiple times" % self.open) - + self._write = write - + # try to open the lock file binary = getattr(os, 'O_BINARY', 0) lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary @@ -313,7 +312,7 @@ def open(self, write=False, stream=False): except OSError: raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) # END handle lock retrieval - + # open actual file if required if self._fd is None: # we could specify exlusive here, as we obtained the lock anyway @@ -325,7 +324,7 @@ def open(self, write=False, stream=False): raise # END handle lockfile # END open descriptor for reading - + if stream: # need delayed import from stream import FDStream @@ -333,33 +332,33 @@ def open(self, write=False, stream=False): else: return self._fd # END handle stream - + def commit(self): - """When done writing, call this function to commit your changes into the - actual file. + """When done writing, call this function to commit your changes into the + actual file. The file descriptor will be closed, and the lockfile handled. - + **Note** can be called multiple times""" self._end_writing(successful=True) - + def rollback(self): - """Abort your operation without any changes. The file descriptor will be + """Abort your operation without any changes. The file descriptor will be closed, and the lock released. - + **Note** can be called multiple times""" self._end_writing(successful=False) - + def _end_writing(self, successful=True): """Handle the lock according to the write mode """ if self._write is None: raise AssertionError("Cannot end operation if it wasn't started yet") - + if self._fd is None: return - + os.close(self._fd) self._fd = None - + lockfile = self._lockfilepath() if self._write and successful: # on windows, rename does not silently overwrite the existing one @@ -369,7 +368,7 @@ def _end_writing(self, successful=True): # END remove if exists # END win32 special handling os.rename(lockfile, self._filepath) - + # assure others can at least read the file - the tmpfile left it at rw-- # We may also write that file, on windows that boils down to a remove- # protection as well From b6c493deb2341fb843d71b66b2aa23078638755c Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 15:41:15 -0400 Subject: [PATCH 0235/1392] Pick off the low hanging fruit This fixes most of the import errors that came from using the implicit relative imports that Python 2 supports. This also fixes the use of `xrange`, which has replaced `range` in Python 3. The same has happened for `izip`, which is also being aliased. The octal number syntax changed in Python 3, so we are now converting from strings using the `int` built-in function, which will produce the same output across both versions of Python. --- gitdb/__init__.py | 7 ++- gitdb/base.py | 18 +++---- gitdb/db/__init__.py | 13 +++-- gitdb/db/base.py | 1 + gitdb/db/git.py | 18 +++---- gitdb/db/loose.py | 20 ++++---- gitdb/db/mem.py | 23 ++++++--- gitdb/db/pack.py | 14 +++--- gitdb/db/ref.py | 10 ++-- gitdb/exc.py | 2 +- gitdb/fun.py | 21 +++++++-- gitdb/pack.py | 94 ++++++++++++++++++++----------------- gitdb/stream.py | 43 ++++++++++------- gitdb/test/__init__.py | 2 +- gitdb/test/db/lib.py | 12 ++++- gitdb/test/db/test_git.py | 2 +- gitdb/test/db/test_loose.py | 2 +- gitdb/test/db/test_mem.py | 2 +- gitdb/test/db/test_pack.py | 2 +- gitdb/test/db/test_ref.py | 2 +- gitdb/test/lib.py | 11 ++++- gitdb/test/test_base.py | 2 +- gitdb/test/test_example.py | 12 +++-- gitdb/test/test_pack.py | 23 +++++---- gitdb/test/test_stream.py | 16 +++---- gitdb/test/test_util.py | 2 +- gitdb/util.py | 18 +++---- 27 files changed, 228 insertions(+), 164 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 847269a33..66d1b1c40 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -32,7 +32,6 @@ def _init_externals(): # default imports -from db import * -from base import * -from stream import * - +from gitdb.base import * +from gitdb.db import * +from gitdb.stream import * diff --git a/gitdb/base.py b/gitdb/base.py index a673c2376..1eb423284 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -3,15 +3,15 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with basic data structures - they are designed to be lightweight and fast""" -from util import ( - bin_to_hex, - zlib - ) - -from fun import ( - type_id_to_type_map, - type_to_type_id_map - ) +from gitdb.util import ( + bin_to_hex, + zlib +) + +from gitdb.fun import ( + type_id_to_type_map, + type_to_type_id_map +) __all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', 'OStream', 'OPackStream', 'ODeltaPackStream', diff --git a/gitdb/db/__init__.py b/gitdb/db/__init__.py index e5935b7c2..0a2a46a64 100644 --- a/gitdb/db/__init__.py +++ b/gitdb/db/__init__.py @@ -3,10 +3,9 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from base import * -from loose import * -from mem import * -from pack import * -from git import * -from ref import * - +from gitdb.db.base import * +from gitdb.db.loose import * +from gitdb.db.mem import * +from gitdb.db.pack import * +from gitdb.db.git import * +from gitdb.db.ref import * diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 0eef1e5d5..85df324f7 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -20,6 +20,7 @@ ) from itertools import chain +from functools import reduce __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') diff --git a/gitdb/db/git.py b/gitdb/db/git.py index 6e6ec5d1f..5c74a2049 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -2,15 +2,15 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from base import ( - CompoundDB, - ObjectDBW, - FileDBBase - ) - -from loose import LooseObjectDB -from pack import PackedDB -from ref import ReferenceDB +from gitdb.db.base import ( + CompoundDB, + ObjectDBW, + FileDBBase +) + +from gitdb.db.loose import LooseObjectDB +from gitdb.db.pack import PackedDB +from gitdb.db.ref import ReferenceDB from gitdb.util import LazyMixin from gitdb.exc import ( diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 4ebca84d3..ac1b9d1d6 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -2,11 +2,11 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from base import ( - FileDBBase, - ObjectDBR, - ObjectDBW - ) +from gitdb.db.base import ( + FileDBBase, + ObjectDBR, + ObjectDBW +) from gitdb.exc import ( @@ -69,11 +69,11 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): # On windows we need to keep it writable, otherwise it cannot be removed # either - new_objects_mode = 0444 + new_objects_mode = int("444", 8) if os.name == 'nt': - new_objects_mode = 0644 - - + new_objects_mode = int("644", 8) + + def __init__(self, root_path): super(LooseObjectDB, self).__init__(root_path) self._hexsha_to_file = dict() @@ -133,7 +133,7 @@ def _map_loose_object(self, sha): db_path = self.db_path(self.object_path(bin_to_hex(sha))) try: return file_contents_ro_filepath(db_path, flags=self._fd_open_flags) - except OSError,e: + except OSError as e: if e.errno != ENOENT: # try again without noatime try: diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index e4fba94b3..3847c34df 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -3,11 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains the MemoryDatabase implementation""" -from loose import LooseObjectDB -from base import ( - ObjectDBR, - ObjectDBW - ) +from gitdb.db.loose import LooseObjectDB +from gitdb.db.base import ( + ObjectDBR, + ObjectDBW +) from gitdb.base import ( OStream, @@ -19,7 +19,18 @@ UnsupportedOperation ) -from cStringIO import StringIO +from gitdb.stream import ( + ZippedStoreShaWriter, + DecompressMemMapReader, +) + +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO __all__ = ("MemoryDB", ) diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 09f811847..eca02bbff 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -3,11 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing a database to deal with packs""" -from base import ( - FileDBBase, - ObjectDBR, - CachingDB - ) +from gitdb.db.base import ( + FileDBBase, + ObjectDBR, + CachingDB +) from gitdb.util import LazyMixin @@ -19,6 +19,8 @@ from gitdb.pack import PackEntity +from functools import reduce + import os import glob @@ -104,7 +106,7 @@ def sha_iter(self): for entity in self.entities(): index = entity.index() sha_by_index = index.sha - for index in xrange(index.size()): + for index in range(index.size()): yield sha_by_index(index) # END for each index # END for each entity diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 368ab9a61..748f7c145 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -2,9 +2,9 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from base import ( - CompoundDB, - ) +from gitdb.db.base import ( + CompoundDB, +) import os __all__ = ('ReferenceDB', ) @@ -33,7 +33,7 @@ def _update_dbs_from_ref_file(self): dbcls = self.ObjectDBCls if dbcls is None: # late import - from git import GitDB + from gitdb.db.git import GitDB dbcls = GitDB # END get db type @@ -68,7 +68,7 @@ def _update_dbs_from_ref_file(self): db.databases() # END verification self._dbs.append(db) - except Exception, e: + except Exception: # ignore invalid paths or issues pass # END for each path to add diff --git a/gitdb/exc.py b/gitdb/exc.py index 47fc80912..73f84d299 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -3,7 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with common exceptions""" -from util import to_hex_sha +from gitdb.util import to_hex_sha class ODBError(Exception): """All errors thrown by the object database""" diff --git a/gitdb/fun.py b/gitdb/fun.py index 9e5c44a6c..ce55438d1 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -6,17 +6,28 @@ Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" -from exc import ( +from gitdb.exc import ( BadObjectType - ) +) -from util import zlib +from gitdb.util import zlib decompressobj = zlib.decompressobj import mmap -from itertools import islice, izip +from itertools import islice + +try: + from itertools import izip +except ImportError: + izip = zip -from cStringIO import StringIO +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO # INVARIANTS OFS_DELTA = 6 diff --git a/gitdb/pack.py b/gitdb/pack.py index 4a0badccb..aea0d1e5a 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -4,31 +4,32 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains PackIndexFile and PackFile implementations""" from gitdb.exc import ( - BadObject, - UnsupportedOperation, - ParseError - ) -from util import ( - zlib, - mman, - LazyMixin, - unpack_from, - bin_to_hex, - ) + BadObject, + UnsupportedOperation, + ParseError +) -from fun import ( - create_pack_object_header, - pack_object_header_info, - is_equal_canonical_sha, - type_id_to_type_map, - write_object, - stream_copy, - chunk_size, - delta_types, - OFS_DELTA, - REF_DELTA, - msb_size - ) +from gitdb.util import ( + zlib, + mman, + LazyMixin, + unpack_from, + bin_to_hex, +) + +from gitdb.fun import ( + create_pack_object_header, + pack_object_header_info, + is_equal_canonical_sha, + type_id_to_type_map, + write_object, + stream_copy, + chunk_size, + delta_types, + OFS_DELTA, + REF_DELTA, + msb_size +) try: from _perf import PackIndexFile_sha_to_index @@ -36,22 +37,23 @@ pass # END try c module -from base import ( # Amazing ! - OInfo, - OStream, - OPackInfo, - OPackStream, - ODeltaStream, - ODeltaPackInfo, - ODeltaPackStream, - ) -from stream import ( - DecompressMemMapReader, - DeltaApplyReader, - Sha1Writer, - NullStream, - FlexibleSha1Writer - ) +from gitdb.base import ( # Amazing ! + OInfo, + OStream, + OPackInfo, + OPackStream, + ODeltaStream, + ODeltaPackInfo, + ODeltaPackStream, +) + +from gitdb.stream import ( + DecompressMemMapReader, + DeltaApplyReader, + Sha1Writer, + NullStream, + FlexibleSha1Writer +) from struct import ( pack, @@ -60,7 +62,11 @@ from binascii import crc32 -from itertools import izip +try: + from itertools import izip +except ImportError: + izip = zip + import tempfile import array import os @@ -200,7 +206,7 @@ def write(self, pack_sha, write): for t in self._objs: tmplist[ord(t[0][0])] += 1 #END prepare fanout - for i in xrange(255): + for i in range(255): v = tmplist[i] sha_write(pack('>L', v)) tmplist[i+1] += v @@ -407,7 +413,7 @@ def offsets(self): a.byteswap() return a else: - return tuple(self.offset(index) for index in xrange(self.size())) + return tuple(self.offset(index) for index in range(self.size())) # END handle version def sha_to_index(self, sha): @@ -694,7 +700,7 @@ def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" _sha = self._index.sha _object = self._object - for index in xrange(self._index.size()): + for index in range(self._index.size()): yield _object(_sha(index), as_stream, index) # END for each index diff --git a/gitdb/stream.py b/gitdb/stream.py index b21c39c9d..52b54af50 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -3,28 +3,35 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from cStringIO import StringIO +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + import errno import mmap import os -from fun import ( - msb_size, - stream_copy, - apply_delta_data, - connect_deltas, - DeltaChunkList, - delta_types - ) - -from util import ( - allocate_memory, - LazyMixin, - make_sha, - write, - close, - zlib - ) +from gitdb.fun import ( + msb_size, + stream_copy, + apply_delta_data, + connect_deltas, + DeltaChunkList, + delta_types +) + +from gitdb.util import ( + allocate_memory, + LazyMixin, + make_sha, + write, + close, + zlib +) has_perf_mod = False try: diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index e84e503b6..ca104c0c5 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -9,7 +9,7 @@ def _init_pool(): """Assure the pool is actually threaded""" size = 2 - print "Setting ThreadPool to %i" % size + print("Setting ThreadPool to %i" % size) gitdb.util.pool.set_size(size) diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index dc89039dd..18b22ff21 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -23,7 +23,15 @@ from gitdb.typ import str_blob_type from async import IteratorReader -from cStringIO import StringIO + +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + from struct import pack @@ -40,7 +48,7 @@ def _assert_object_writing_simple(self, db): # write a bunch of objects and query their streams and info null_objs = db.size() ni = 250 - for i in xrange(ni): + for i in range(ni): data = pack(">L", i) istream = IStream(str_blob_type, len(data), StringIO(data)) new_istream = db.store(istream) diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 4894c6a79..cce2b9c09 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.exc import BadObject from gitdb.db import GitDB from gitdb.base import OStream, OInfo diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index e295db563..5e42b639a 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index ac9bc34e9..9235b21d3 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.db import ( MemoryDB, LooseObjectDB diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 0d9110a01..f5a4dcb92 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.db import PackedDB from gitdb.test.lib import fixture_path diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 086446823..752c31de5 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -2,7 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from lib import * +from gitdb.test.db.lib import * from gitdb.db import ReferenceDB from gitdb.util import ( diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 685af2fad..3ac7142c1 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -16,7 +16,14 @@ import sys import random from array import array -from cStringIO import StringIO + +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO import glob import unittest @@ -109,7 +116,7 @@ def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" actual_size = size_in_bytes / 4 - producer = xrange(actual_size) + producer = range(actual_size) if randomize: producer = list(producer) random.shuffle(producer) diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 76b4d2709..4cca7dabe 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -3,7 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" -from lib import ( +from gitdb.test.lib import ( TestBase, DummyStream, DeriveTest, diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index f45063b1e..f57cc5029 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -3,12 +3,18 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with examples from the tutorial section of the docs""" -from lib import * +from gitdb.test.lib import * from gitdb import IStream from gitdb.db import LooseObjectDB from gitdb.util import pool - -from cStringIO import StringIO + +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO from async import IteratorReader diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index f28aef4d4..bcda3cfb8 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -3,12 +3,13 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test everything about packs reading and writing""" -from lib import ( - TestBase, - with_rw_directory, - with_packs_rw, - fixture_path - ) +from gitdb.test.lib import ( + TestBase, + with_rw_directory, + with_packs_rw, + fixture_path +) + from gitdb.stream import DeltaApplyReader from gitdb.pack import ( @@ -25,7 +26,13 @@ from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha -from itertools import izip, chain +from itertools import chain + +try: + from itertools import izip +except ImportError: + izip = zip + from nose import SkipTest import os @@ -57,7 +64,7 @@ def _assert_index_file(self, index, version, size): assert len(index.offsets()) == size # get all data of all objects - for oidx in xrange(index.size()): + for oidx in range(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 8360ea36c..53aa8e21d 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -3,14 +3,14 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" -from lib import ( - TestBase, - DummyStream, - Sha1Writer, - make_bytes, - make_object, - fixture_path - ) +from gitdb.test.lib import ( + TestBase, + DummyStream, + Sha1Writer, + make_bytes, + make_object, + fixture_path +) from gitdb import * from gitdb.util import ( diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index ed69f0d1f..4672dd604 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -6,7 +6,7 @@ import tempfile import os -from lib import TestBase +from gitdb.test.lib import TestBase from gitdb.util import ( to_hex_sha, to_bin_sha, diff --git a/gitdb/util.py b/gitdb/util.py index b167b4d12..3dcf3d7dc 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -8,13 +8,13 @@ import sys import errno -from cStringIO import StringIO - -# in py 2.4, StringIO is only StringI, without write support. -# Hence we must use the python implementation for this -if sys.version_info[1] < 5: - from StringIO import StringIO -# END handle python 2.4 +try: + from cStringIO import StringIO +except ImportError: + try: + from StringIO import StringIO + except ImportError: + from io import StringIO try: import async.mod.zlib as zlib @@ -303,7 +303,7 @@ def open(self, write=False, stream=False): binary = getattr(os, 'O_BINARY', 0) lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary try: - fd = os.open(self._lockfilepath(), lockmode, 0600) + fd = os.open(self._lockfilepath(), lockmode, int("600", 8)) if not write: os.close(fd) else: @@ -372,7 +372,7 @@ def _end_writing(self, successful=True): # assure others can at least read the file - the tmpfile left it at rw-- # We may also write that file, on windows that boils down to a remove- # protection as well - chmod(self._filepath, 0644) + chmod(self._filepath, int("644", 8)) else: # just delete the file so far, we failed os.remove(lockfile) From 5bfa7e6cb0872c815720f7e861393125ad0855e7 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 15:45:51 -0400 Subject: [PATCH 0236/1392] Temporarily switch out async for testing This will be switched back when the pull request for Python 3 support has been merged into the central async repository. --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 978105388..c28387570 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "async"] path = gitdb/ext/async - url = https://github.com/gitpython-developers/async.git + url = https://github.com/kevin-brown/async.git [submodule "smmap"] path = gitdb/ext/smmap url = https://github.com/Byron/smmap.git From 85f2b9baf1c6a5738967f8e77d2a65a1a842bde3 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 15:48:03 -0400 Subject: [PATCH 0237/1392] Test against Python 3.4 If it works in Python 3.3, it should also work in Python 3.4. Considering it is the latest stable release, gitdb should be tested against it. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ff263f20d..cf1d13666 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - "2.6" - "2.7" - "3.3" + - "3.4" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) script: nosetests From b881134ec816d2a54f6e8deced8db25b4bd5baa7 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 16:10:54 -0400 Subject: [PATCH 0238/1392] Convert strings to bytes for PY3 In Python 3, the default string type is now the Python 2 unicode strings. The unicode strings cannot be converted to a byte stream, so we have to convert it before writing to the streams. --- gitdb/test/lib.py | 6 +++--- gitdb/test/test_stream.py | 9 ++++----- gitdb/test/test_util.py | 13 +++++++------ gitdb/util.py | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 3ac7142c1..f52cf79d4 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -54,7 +54,7 @@ def wrapper(self): try: return func(self, path) except Exception: - print >> sys.stderr, "Test %s.%s failed, output is at %r" % (type(self).__name__, func.__name__, path) + sys.stderr.write("Test %s.%s failed, output is at %r\n" % (type(self).__name__, func.__name__, path)) keep = True raise finally: @@ -115,7 +115,7 @@ def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" - actual_size = size_in_bytes / 4 + actual_size = size_in_bytes // 4 producer = range(actual_size) if randomize: producer = list(producer) @@ -127,7 +127,7 @@ def make_bytes(size_in_bytes, randomize=False): def make_object(type, data): """:return: bytes resembling an uncompressed object""" odata = "blob %i\0" % len(data) - return odata + data + return odata.encode("ascii") + data def make_memory_file(size_in_bytes, randomize=False): """:return: tuple(size_of_stream, stream) diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 53aa8e21d..f6eb371bd 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -3,6 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" + from gitdb.test.lib import ( TestBase, DummyStream, @@ -16,20 +17,18 @@ from gitdb.util import ( NULL_HEX_SHA, hex_to_bin - ) +) from gitdb.util import zlib from gitdb.typ import ( str_blob_type - ) +) import time import tempfile import os - - class TestStream(TestBase): """Test stream classes""" @@ -45,7 +44,7 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) # read in small steps - ss = len(cdata) / ns + ss = len(cdata) // ns for i in range(ns): data = stream.read(ss) chunk = cdata[i*ss:(i+1)*ss] diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 4672dd604..ec9a86ce7 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -5,6 +5,7 @@ """Test for object db""" import tempfile import os +import sys from gitdb.test.lib import TestBase from gitdb.util import ( @@ -19,14 +20,14 @@ class TestUtils(TestBase): def test_basics(self): assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA assert len(to_bin_sha(NULL_HEX_SHA)) == 20 - assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA + assert to_hex_sha(to_bin_sha(NULL_HEX_SHA)) == NULL_HEX_SHA.encode("ascii") def _cmp_contents(self, file_path, data): # raise if data from file at file_path # does not match data string fp = open(file_path, "rb") try: - assert fp.read() == data + assert fp.read() == data.encode("ascii") finally: fp.close() @@ -35,7 +36,7 @@ def test_lockedfd(self): orig_data = "hello" new_data = "world" my_file_fp = open(my_file, "wb") - my_file_fp.write(orig_data) + my_file_fp.write(orig_data.encode("ascii")) my_file_fp.close() try: @@ -53,7 +54,7 @@ def test_lockedfd(self): assert os.path.isfile(lockfilepath) # write data and fail - os.write(wfd, new_data) + os.write(wfd, new_data.encode("ascii")) lfd.rollback() assert lfd._fd is None self._cmp_contents(my_file, orig_data) @@ -66,7 +67,7 @@ def test_lockedfd(self): # test reading lfd = LockedFD(my_file) rfd = lfd.open(write=False) - assert os.read(rfd, len(orig_data)) == orig_data + assert os.read(rfd, len(orig_data)) == orig_data.encode("ascii") assert os.path.isfile(lockfilepath) # deletion rolls back @@ -83,7 +84,7 @@ def test_lockedfd(self): # another one fails self.failUnlessRaises(IOError, olfd.open) - wfdstream.write(new_data) + wfdstream.write(new_data.encode("ascii")) lfd.commit() assert not os.path.isfile(lockfilepath) self._cmp_contents(my_file, new_data) diff --git a/gitdb/util.py b/gitdb/util.py index 3dcf3d7dc..5ea26d5bd 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -327,7 +327,7 @@ def open(self, write=False, stream=False): if stream: # need delayed import - from stream import FDStream + from gitdb.stream import FDStream return FDStream(self._fd) else: return self._fd From 01e40b5e02e90ccac06e3b0ec0adf1f8f4e48ebd Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 17:29:58 -0400 Subject: [PATCH 0239/1392] Use memoryview instead of buffer This uses memoryview by default, which is supported in Python 3 and Python 2.7, but not Python 2.6, and falls back to the old `buffer` type in Python 2.6 and when the memoryview does not support the type, such as when mmap instaces are passed in. --- gitdb/stream.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index 52b54af50..a099eeba0 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -270,7 +270,10 @@ def read(self, size=-1): # END adjust winsize # takes a slice, but doesn't copy the data, it says ... - indata = buffer(self._m, self._cws, self._cwe - self._cws) + try: + indata = memoryview(self._m)[self._cws:self._cwe].tobytes() + except (NameError, TypeError): + indata = buffer(self._m, self._cws, self._cwe - self._cws) # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) From 0269405121d7ef065f7008c9c033e95e734f029a Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 18:20:48 -0400 Subject: [PATCH 0240/1392] Better handling of bytes This adds a `byte_ord` version of `ord` which will let `bytes` safely pass through in Python 3. `cmp` was also swapped out as it has been dropped in Python 3. --- gitdb/fun.py | 10 +++++----- gitdb/pack.py | 26 ++++++++++++++------------ gitdb/util.py | 15 ++++++++++++++- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index ce55438d1..8f2d46342 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -10,7 +10,7 @@ BadObjectType ) -from gitdb.util import zlib +from gitdb.util import byte_ord, zlib decompressobj = zlib.decompressobj import mmap @@ -411,13 +411,13 @@ def pack_object_header_info(data): The type_id should be interpreted according to the ``type_id_to_type_map`` map The byte-offset specifies the start of the actual zlib compressed datastream :param m: random-access memory, like a string or memory map""" - c = ord(data[0]) # first byte + c = byte_ord(data[0]) # first byte i = 1 # next char to read type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size s = 4 # starting bit-shift size while c & 0x80: - c = ord(data[i]) + c = byte_ord(data[i]) i += 1 size += (c & 0x7f) << s s += 7 @@ -668,12 +668,12 @@ def is_equal_canonical_sha(canonical_length, match, sha1): hence the comparison will only use the last 4 bytes for uneven canonical representations :param match: less than 20 byte sha :param sha1: 20 byte sha""" - binary_length = canonical_length/2 + binary_length = canonical_length // 2 if match[:binary_length] != sha1[:binary_length]: return False if canonical_length - binary_length and \ - (ord(match[-1]) ^ ord(sha1[len(match)-1])) & 0xf0: + (byte_ord(match[-1]) ^ byte_ord(sha1[len(match)-1])) & 0xf0: return False # END handle uneven canonnical length return True diff --git a/gitdb/pack.py b/gitdb/pack.py index aea0d1e5a..4a9ee9ffa 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -15,6 +15,7 @@ LazyMixin, unpack_from, bin_to_hex, + byte_ord, ) from gitdb.fun import ( @@ -421,7 +422,7 @@ def sha_to_index(self, sha): :return: index usable with the ``offset`` or ``entry`` method, or None if the sha was not found in this pack index :param sha: 20 byte sha to lookup""" - first_byte = ord(sha[0]) + first_byte = byte_ord(sha[0]) get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: @@ -430,11 +431,11 @@ def sha_to_index(self, sha): # bisect until we have the sha while lo < hi: - mid = (lo + hi) / 2 - c = cmp(sha, get_sha(mid)) - if c < 0: + mid = (lo + hi) // 2 + mid_sha = get_sha(mid) + if sha < mid_sha: hi = mid - elif not c: + elif sha == mid_sha: return mid else: lo = mid + 1 @@ -453,7 +454,8 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): if len(partial_bin_sha) < 2: raise ValueError("Require at least 2 bytes of partial sha") - first_byte = ord(partial_bin_sha[0]) + first_byte = byte_ord(partial_bin_sha[0]) + get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: @@ -461,15 +463,15 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): hi = self._fanout_table[first_byte] # the upper, right bound of the bisection # fill the partial to full 20 bytes - filled_sha = partial_bin_sha + '\0'*(20 - len(partial_bin_sha)) + filled_sha = partial_bin_sha + '\0'.encode("ascii") * (20 - len(partial_bin_sha)) # find lowest while lo < hi: - mid = (lo + hi) / 2 - c = cmp(filled_sha, get_sha(mid)) - if c < 0: + mid = (lo + hi) // 2 + mid_sha = get_sha(mid) + if filled_sha < mid_sha: hi = mid - elif not c: + elif filled_sha == mid_sha: # perfect match lo = mid break @@ -482,7 +484,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): cur_sha = get_sha(lo) if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): next_sha = None - if lo+1 < self.size(): + if lo + 1 < self.size(): next_sha = get_sha(lo+1) if next_sha and next_sha == cur_sha: raise AmbiguousObjectName(partial_bin_sha) diff --git a/gitdb/util.py b/gitdb/util.py index 5ea26d5bd..ed2dc3723 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -118,14 +118,27 @@ def __getitem__(self, i): def __getslice__(self, start, end): return self.getvalue()[start:end] +def byte_ord(b): + """ + Return the integer representation of the byte string. This supports Python + 3 byte arrays as well as standard strings. + """ + try: + return ord(b) + except TypeError: + return b + #} END compatibility stuff ... #{ Routines -def make_sha(source=''): +def make_sha(source=None): """A python2.4 workaround for the sha/hashlib module fiasco **Note** From the dulwich project """ + if source is None: + source = "".encode("ascii") + try: return hashlib.sha1(source) except NameError: From d8405ee00bfc1ebdae7c41b45f8c374902a3d2b4 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 19:55:00 -0400 Subject: [PATCH 0241/1392] More bytes handling --- gitdb/db/base.py | 10 ++++++++++ gitdb/fun.py | 2 +- gitdb/pack.py | 6 +++++- gitdb/stream.py | 4 ++-- gitdb/test/db/test_ref.py | 2 +- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 85df324f7..aa7a7ee30 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -22,6 +22,8 @@ from itertools import chain from functools import reduce +import sys + __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') @@ -176,6 +178,14 @@ def db_path(self, rela_path): """ :return: the given relative path relative to our database root, allowing to pontentially access datafiles""" + if sys.version_info[0] == 3: + text_type = str + else: + text_type = basestring + + if not isinstance(rela_path, text_type): + rela_path = rela_path.decode("utf-8") + return join(self._root_path, rela_path) #} END interface diff --git a/gitdb/fun.py b/gitdb/fun.py index 8f2d46342..9baeac3ec 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -402,7 +402,7 @@ def loose_object_header_info(m): :param m: memory map from which to read the compressed object data""" decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) - type_name, size = hdr[:hdr.find("\0")].split(" ") + type_name, size = hdr[:hdr.find("\0".encode("ascii"))].split(" ".encode("ascii")) return type_name, int(size) def pack_object_header_info(data): diff --git a/gitdb/pack.py b/gitdb/pack.py index 4a9ee9ffa..ad5eccb8b 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -121,7 +121,11 @@ def pack_object_at(cursor, offset, as_stream): abs_data_offset = offset + total_rela_offset if as_stream: - stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) + try: + buff = memoryview(data)[total_rela_offset:].tobytes() + except (NameError, TypeError): + buff = buffer(data, total_rela_offset) + stream = DecompressMemMapReader(buff, False, uncomp_size) if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: diff --git a/gitdb/stream.py b/gitdb/stream.py index a099eeba0..75993db51 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -105,8 +105,8 @@ def _parse_header_info(self): maxb = 512 # should really be enough, cgit uses 8192 I believe self._s = maxb hdr = self.read(maxb) - hdrend = hdr.find("\0") - type, size = hdr[:hdrend].split(" ") + hdrend = hdr.find("\0".encode("ascii")) + type, size = hdr[:hdrend].split(" ".encode("ascii")) size = int(size) self._s = size diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 752c31de5..e303fe2f8 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -19,7 +19,7 @@ def make_alt_file(self, alt_path, alt_list): The list can be empty""" alt_file = open(alt_path, "wb") for alt in alt_list: - alt_file.write(alt + "\n") + alt_file.write(alt.encode("utf-8") + "\n".encode("ascii")) alt_file.close() @with_rw_directory From f0b3e7bcc5278208294f40aa580ccb378ed1c165 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 20:14:24 -0400 Subject: [PATCH 0242/1392] Bytes for everyone! --- gitdb/db/loose.py | 8 ++++---- gitdb/stream.py | 2 +- gitdb/test/db/test_ref.py | 3 +-- gitdb/test/test_stream.py | 4 ++-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index ac1b9d1d6..656841651 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -16,10 +16,10 @@ ) from gitdb.stream import ( - DecompressMemMapReader, - FDCompressedSha1Writer, - FDStream, - Sha1Writer + DecompressMemMapReader, + FDCompressedSha1Writer, + FDStream, + Sha1Writer ) from gitdb.base import ( diff --git a/gitdb/stream.py b/gitdb/stream.py index 75993db51..1bd7fe2b2 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -551,7 +551,7 @@ def __init__(self): def write(self, data): """:raise IOError: If not all bytes could be written - :return: lenght of incoming data""" + :return: length of incoming data""" self.sha1.update(data) return len(data) diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index e303fe2f8..a1387ee26 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -24,7 +24,7 @@ def make_alt_file(self, alt_path, alt_list): @with_rw_directory def test_writing(self, path): - NULL_BIN_SHA = '\0' * 20 + NULL_BIN_SHA = '\0'.encode("ascii") * 20 alt_path = os.path.join(path, 'alternates') rdb = ReferenceDB(alt_path) @@ -35,7 +35,6 @@ def test_writing(self, path): # try empty, non-existing assert not rdb.has_object(NULL_BIN_SHA) - # setup alternate file # add two, one is invalid own_repo_path = fixture_path('../../../.git/objects') # use own repo diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index f6eb371bd..f409f1788 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -111,13 +111,13 @@ def test_decompress_reader(self): def test_sha_writer(self): writer = Sha1Writer() - assert 2 == writer.write("hi") + assert 2 == writer.write("hi".encode("ascii")) assert len(writer.sha(as_hex=1)) == 40 assert len(writer.sha(as_hex=0)) == 20 # make sure it does something ;) prev_sha = writer.sha() - writer.write("hi again") + writer.write("hi again".encode("ascii")) assert writer.sha() != prev_sha def test_compressed_writer(self): From a19a169ffd81e21f472dee8a9a38ac1c9fab9bd7 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 21:04:47 -0400 Subject: [PATCH 0243/1392] Can't compare memoryview instances, convert to bytes --- gitdb/pack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index ad5eccb8b..d8f32c1c2 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -118,7 +118,6 @@ def pack_object_at(cursor, offset, as_stream): # assume its a base object total_rela_offset = data_rela_offset # END handle type id - abs_data_offset = offset + total_rela_offset if as_stream: try: @@ -129,6 +128,8 @@ def pack_object_at(cursor, offset, as_stream): if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: + if hasattr(delta_info, "tobytes"): + delta_info = delta_info.tobytes() return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) else: if delta_info is None: From 087803ee30456c4942d9c18d82c1d686eb081a27 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 21:49:51 -0400 Subject: [PATCH 0244/1392] Making a bit of progress... This changes the internals to use BytesIO over StringIO, which fixed a few of the failing tests in Python 3. We are only importing from `io` now, instead of the entire chain, as this is available in Python 2.6+. --- gitdb/db/mem.py | 8 +------- gitdb/fun.py | 10 ++-------- gitdb/stream.py | 26 ++++++++++---------------- gitdb/test/db/lib.py | 10 ++-------- gitdb/test/lib.py | 8 +------- gitdb/test/test_example.py | 8 +------- gitdb/test/test_stream.py | 4 ++-- gitdb/util.py | 8 +------- 8 files changed, 20 insertions(+), 62 deletions(-) diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 3847c34df..1a2378f08 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -24,13 +24,7 @@ DecompressMemMapReader, ) -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO __all__ = ("MemoryDB", ) diff --git a/gitdb/fun.py b/gitdb/fun.py index 9baeac3ec..1d835ab35 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -21,13 +21,7 @@ except ImportError: izip = zip -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO # INVARIANTS OFS_DELTA = 6 @@ -453,7 +447,7 @@ def msb_size(data, offset=0): l = len(data) hit_msb = False while i < l: - c = ord(data[i+offset]) + c = byte_ord(data[i+offset]) size |= (c & 0x7f) << i*7 i += 1 if not c & 0x80: diff --git a/gitdb/stream.py b/gitdb/stream.py index 1bd7fe2b2..9b451506e 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -3,13 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import BytesIO, StringIO import errno import mmap @@ -106,20 +100,20 @@ def _parse_header_info(self): self._s = maxb hdr = self.read(maxb) hdrend = hdr.find("\0".encode("ascii")) - type, size = hdr[:hdrend].split(" ".encode("ascii")) + typ, size = hdr[:hdrend].split(" ".encode("ascii")) size = int(size) self._s = size # adjust internal state to match actual header length that we ignore # The buffer will be depleted first on future reads self._br = 0 - hdrend += 1 # count terminating \0 - self._buf = StringIO(hdr[hdrend:]) + hdrend += 1 + self._buf = BytesIO(hdr[hdrend:]) self._buflen = len(hdr) - hdrend self._phi = True - return type, size + return typ.decode("ascii"), size #{ Interface @@ -133,8 +127,8 @@ def new(self, m, close_on_deletion=False): :param close_on_deletion: if True, the memory map will be closed once we are being deleted""" inst = DecompressMemMapReader(m, close_on_deletion, 0) - type, size = inst._parse_header_info() - return type, size, inst + typ, size = inst._parse_header_info() + return typ, size, inst def data(self): """:return: random access compatible data we are working on""" @@ -211,14 +205,14 @@ def read(self, size=-1): # END clamp size if size == 0: - return str() + return bytes() # END handle depletion # deplete the buffer, then just continue using the decompress object # which has an own buffer. We just need this to transparently parse the # header from the zlib stream - dat = str() + dat = bytes() if self._buf: if self._buflen >= size: # have enough data @@ -588,7 +582,7 @@ class ZippedStoreShaWriter(Sha1Writer): __slots__ = ('buf', 'zip') def __init__(self): Sha1Writer.__init__(self) - self.buf = StringIO() + self.buf = BytesIO() self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) def __getattr__(self, attr): diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 18b22ff21..15fbf3fc3 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -24,13 +24,7 @@ from async import IteratorReader -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO from struct import pack @@ -41,7 +35,7 @@ class TestDBBase(TestBase): """Base class providing testing routines on databases""" # data - two_lines = "1234\nhello world" + two_lines = "1234\nhello world".encode("ascii") all_data = (two_lines, ) def _assert_object_writing_simple(self, db): diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index f52cf79d4..75342f173 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -17,13 +17,7 @@ import random from array import array -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO import glob import unittest diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index f57cc5029..c714f107f 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -8,13 +8,7 @@ from gitdb.db import LooseObjectDB from gitdb.util import pool -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO from async import IteratorReader diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index f409f1788..92755d9aa 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -82,9 +82,9 @@ def test_decompress_reader(self): if with_size: # need object data zdata = zlib.compress(make_object(str_blob_type, cdata)) - type, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) + typ, size, reader = DecompressMemMapReader.new(zdata, close_on_deletion) assert size == len(cdata) - assert type == str_blob_type + assert typ == str_blob_type # even if we don't set the size, it will be set automatically on first read test_reader = DecompressMemMapReader(zdata, close_on_deletion=False) diff --git a/gitdb/util.py b/gitdb/util.py index ed2dc3723..30c7008f4 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -8,13 +8,7 @@ import sys import errno -try: - from cStringIO import StringIO -except ImportError: - try: - from StringIO import StringIO - except ImportError: - from io import StringIO +from io import StringIO try: import async.mod.zlib as zlib From 0cf09d3310cba7f33b9ebc9badf61ab721d12857 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Sun, 13 Jul 2014 22:15:09 -0400 Subject: [PATCH 0245/1392] Fix Python 2 failures --- gitdb/db/loose.py | 4 ++-- gitdb/db/mem.py | 4 ++-- gitdb/fun.py | 2 +- gitdb/test/db/lib.py | 11 ++++++----- gitdb/test/test_example.py | 6 +++--- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 656841651..1b8fe643c 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -161,8 +161,8 @@ def set_ostream(self, stream): def info(self, sha): m = self._map_loose_object(sha) try: - type, size = loose_object_header_info(m) - return OInfo(sha, type, size) + typ, size = loose_object_header_info(m) + return OInfo(sha, typ, size) finally: m.close() # END assure release of system resources diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 1a2378f08..efd85af27 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -24,7 +24,7 @@ DecompressMemMapReader, ) -from io import StringIO +from io import BytesIO __all__ = ("MemoryDB", ) @@ -104,7 +104,7 @@ def stream_copy(self, sha_iter, odb): ostream = self.stream(sha) # compressed data including header - sio = StringIO(ostream.stream.data()) + sio = BytesIO(ostream.stream.data()) istream = IStream(ostream.type, ostream.size, sio, sha) odb.store(istream) diff --git a/gitdb/fun.py b/gitdb/fun.py index 1d835ab35..69e9826d1 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -397,7 +397,7 @@ def loose_object_header_info(m): decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) type_name, size = hdr[:hdr.find("\0".encode("ascii"))].split(" ".encode("ascii")) - return type_name, int(size) + return type_name.decode("ascii"), int(size) def pack_object_header_info(data): """ diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 15fbf3fc3..a6cdbbe65 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -24,7 +24,7 @@ from async import IteratorReader -from io import StringIO +from io import BytesIO from struct import pack @@ -44,7 +44,7 @@ def _assert_object_writing_simple(self, db): ni = 250 for i in range(ni): data = pack(">L", i) - istream = IStream(str_blob_type, len(data), StringIO(data)) + istream = IStream(str_blob_type, len(data), BytesIO(data)) new_istream = db.store(istream) assert new_istream is istream assert db.has_object(istream.binsha) @@ -82,7 +82,7 @@ def _assert_object_writing(self, db): prev_ostream = db.set_ostream(ostream) assert type(prev_ostream) in ostreams or prev_ostream in ostreams - istream = IStream(str_blob_type, len(data), StringIO(data)) + istream = IStream(str_blob_type, len(data), BytesIO(data)) # store returns same istream instance, with new sha set my_istream = db.store(istream) @@ -132,8 +132,9 @@ def _assert_object_writing_async(self, db): ni = 5000 def istream_generator(offset=0, ni=ni): for data_src in xrange(ni): - data = str(data_src + offset) - yield IStream(str_blob_type, len(data), StringIO(data)) + print(type(data_src), type(offset)) + data = bytes(data_src + offset) + yield IStream(str_blob_type, len(data), BytesIO(data)) # END for each item # END generator utility diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index c714f107f..c644b8849 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -8,7 +8,7 @@ from gitdb.db import LooseObjectDB from gitdb.util import pool -from io import StringIO +from io import BytesIO from async import IteratorReader @@ -33,8 +33,8 @@ def test_base(self): pass # END ignore exception if there are no loose objects - data = "my data" - istream = IStream("blob", len(data), StringIO(data)) + data = "my data".encode("ascii") + istream = IStream("blob", len(data), BytesIO(data)) # the object does not yet have a sha assert istream.binsha is None From 9dc111e1aa8358aa39a35d5a169335bacce53646 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 14 Jul 2014 13:01:54 +0200 Subject: [PATCH 0246/1392] Added sublime-text project Suitable for everyone thanks to relative paths --- .gitignore | 2 + etc/sublime-text/gitdb.sublime-project | 54 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 etc/sublime-text/gitdb.sublime-project diff --git a/.gitignore b/.gitignore index 1e097f6f8..c6247dbb0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ dist/ *.pyc *.o *.so +.noseids +*.sublime-workspace \ No newline at end of file diff --git a/etc/sublime-text/gitdb.sublime-project b/etc/sublime-text/gitdb.sublime-project new file mode 100644 index 000000000..bc0e37f0a --- /dev/null +++ b/etc/sublime-text/gitdb.sublime-project @@ -0,0 +1,54 @@ +{ + "folders": + [ + // GITDB + //////// + { + "follow_symlinks": true, + "path": "../..", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + "gitdb/ext" + ] + }, + // SMMAP + //////// + { + "follow_symlinks": true, + "path": "../../gitdb/ext/smmap", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + ] + }, + // ASYNC + //////// + { + "follow_symlinks": true, + "path": "../../gitdb/ext/async", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + ] + }, + ] +} From 1af4b42a2354acbb53c7956d647655922658fd80 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 14 Jul 2014 13:05:41 +0200 Subject: [PATCH 0247/1392] Added sublime-text project Relative paths will make it work for everyone right away --- .gitignore | 2 ++ etc/sublime-text/smmap.sublime-project | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 etc/sublime-text/smmap.sublime-project diff --git a/.gitignore b/.gitignore index 73b0b6188..01247547d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ dist/ MANIFEST .tox *.egg-info +.noseids +*.sublime-workspace diff --git a/etc/sublime-text/smmap.sublime-project b/etc/sublime-text/smmap.sublime-project new file mode 100644 index 000000000..251ebbd28 --- /dev/null +++ b/etc/sublime-text/smmap.sublime-project @@ -0,0 +1,21 @@ +{ + "folders": + [ + // SMMAP + //////// + { + "follow_symlinks": true, + "path": "../..", + "file_exclude_patterns" : [ + "*.sublime-workspace", + ".git", + ".noseids", + ".coverage" + ], + "folder_exclude_patterns" : [ + ".git", + "cover", + ] + }, + ] +} From 0465cf327d232101b2de69d714a468b7e1a66a74 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Wed, 16 Jul 2014 20:15:31 -0400 Subject: [PATCH 0248/1392] Start up compat and encoding files There were a few things which were being reused consistently for compatibility purposes, such as the `buffer`/`memoryview` functions as well as the `izip` method which needed to be aliased for Python 3. The `buffer` function was taken from `smmap` [1] and reworked slightly to handle the optional third parameter. This also adds a compatibility file dedicated entirely to encoding issues, which seem to be the biggest problem. The main functions were taken in part from the Django project [2] and rewritten slightly because our needs are a bit more narrow. A constants file has been added to consistently handle the constants which are required for the gitdb project in the core and the tests. This is part of a greater plan to reorganize the `util.py` file included in this project. This points the async extension back at the original repository and points it to the latest commit. [1]: https://github.com/Byron/smmap/blob/1af4b42a2354acbb53c7956d647655922658fd80/smmap/util.py#L20-L26 [2]: https://github.com/django/django/blob/b8d255071ead897cf68120cd2fae7c91326ca2cc/django/utils/encoding.py --- .gitmodules | 2 +- gitdb/const.py | 5 +++++ gitdb/db/base.py | 10 ++-------- gitdb/db/loose.py | 4 +++- gitdb/ext/async | 2 +- gitdb/fun.py | 13 +++++++------ gitdb/pack.py | 18 +++++++----------- gitdb/stream.py | 24 ++++++++++++++++++------ gitdb/test/db/lib.py | 8 +++----- gitdb/util.py | 10 +++------- gitdb/utils/__init__.py | 0 gitdb/utils/compat.py | 39 +++++++++++++++++++++++++++++++++++++++ gitdb/utils/encoding.py | 35 +++++++++++++++++++++++++++++++++++ 13 files changed, 124 insertions(+), 46 deletions(-) create mode 100644 gitdb/const.py create mode 100644 gitdb/utils/__init__.py create mode 100644 gitdb/utils/compat.py create mode 100644 gitdb/utils/encoding.py diff --git a/.gitmodules b/.gitmodules index c28387570..978105388 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "async"] path = gitdb/ext/async - url = https://github.com/kevin-brown/async.git + url = https://github.com/gitpython-developers/async.git [submodule "smmap"] path = gitdb/ext/smmap url = https://github.com/Byron/smmap.git diff --git a/gitdb/const.py b/gitdb/const.py new file mode 100644 index 000000000..147f79cb9 --- /dev/null +++ b/gitdb/const.py @@ -0,0 +1,5 @@ +from gitdb.utils.encoding import force_bytes + +NULL_BYTE = force_bytes("\0") +NULL_HEX_SHA = "0" * 40 +NULL_BIN_SHA = NULL_BYTE * 20 diff --git a/gitdb/db/base.py b/gitdb/db/base.py index aa7a7ee30..53a94d249 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -178,15 +178,9 @@ def db_path(self, rela_path): """ :return: the given relative path relative to our database root, allowing to pontentially access datafiles""" - if sys.version_info[0] == 3: - text_type = str - else: - text_type = basestring - - if not isinstance(rela_path, text_type): - rela_path = rela_path.decode("utf-8") + from gitdb.utils.encoding import force_text - return join(self._root_path, rela_path) + return join(self._root_path, force_text(rela_path)) #} END interface diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 1b8fe643c..f66d6275f 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -51,6 +51,8 @@ stream_copy ) +from gitdb.utils.compat import MAXSIZE + import tempfile import mmap import sys @@ -200,7 +202,7 @@ def store(self, istream): if istream.binsha is not None: # copy as much as possible, the actual uncompressed item size might # be smaller than the compressed version - stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size) + stream_copy(istream.read, writer.write, MAXSIZE, self.stream_chunk_size) else: # write object with header, we have to make a new one write_object(istream.type, istream.size, istream.read, writer.write, diff --git a/gitdb/ext/async b/gitdb/ext/async index 339024bfb..3f26b05c2 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 339024bfb1d0a2b091e63d7a7ea23a1c63189f5c +Subproject commit 3f26b05c2f1a079d5807ed15c01b053ee846e745 diff --git a/gitdb/fun.py b/gitdb/fun.py index 69e9826d1..2749f37d0 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -16,10 +16,7 @@ import mmap from itertools import islice -try: - from itertools import izip -except ImportError: - izip = zip +from gitdb.utils.compat import izip from io import StringIO @@ -394,10 +391,14 @@ def loose_object_header_info(m): :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" + from gitdb.const import NULL_BYTE + from gitdb.utils.encoding import force_text + decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) - type_name, size = hdr[:hdr.find("\0".encode("ascii"))].split(" ".encode("ascii")) - return type_name.decode("ascii"), int(size) + type_name, size = hdr[:hdr.find(NULL_BYTE)].split(" ".encode("ascii")) + + return force_text(type_name), int(size) def pack_object_header_info(data): """ diff --git a/gitdb/pack.py b/gitdb/pack.py index d8f32c1c2..4e83ba39d 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -63,10 +63,8 @@ from binascii import crc32 -try: - from itertools import izip -except ImportError: - izip = zip +from gitdb.const import NULL_BYTE +from gitdb.utils.compat import izip, buffer import tempfile import array @@ -90,6 +88,8 @@ def pack_object_at(cursor, offset, as_stream): :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" + from gitdb.utils.encoding import force_bytes + data = cursor.use_region(offset).buffer() type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins @@ -120,16 +120,12 @@ def pack_object_at(cursor, offset, as_stream): # END handle type id abs_data_offset = offset + total_rela_offset if as_stream: - try: - buff = memoryview(data)[total_rela_offset:].tobytes() - except (NameError, TypeError): - buff = buffer(data, total_rela_offset) + buff = buffer(data, total_rela_offset) stream = DecompressMemMapReader(buff, False, uncomp_size) if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: - if hasattr(delta_info, "tobytes"): - delta_info = delta_info.tobytes() + delta_info = force_bytes(delta_info) return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) else: if delta_info is None: @@ -468,7 +464,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): hi = self._fanout_table[first_byte] # the upper, right bound of the bisection # fill the partial to full 20 bytes - filled_sha = partial_bin_sha + '\0'.encode("ascii") * (20 - len(partial_bin_sha)) + filled_sha = partial_bin_sha + NULL_BYTE * (20 - len(partial_bin_sha)) # find lowest while lo < hi: diff --git a/gitdb/stream.py b/gitdb/stream.py index 9b451506e..9e49f50e0 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -27,6 +27,10 @@ zlib ) +from gitdb.const import NULL_BYTE +from gitdb.utils.compat import buffer +from gitdb.utils.encoding import force_bytes, force_text + has_perf_mod = False try: from _perf import apply_delta as c_apply_delta @@ -99,7 +103,7 @@ def _parse_header_info(self): maxb = 512 # should really be enough, cgit uses 8192 I believe self._s = maxb hdr = self.read(maxb) - hdrend = hdr.find("\0".encode("ascii")) + hdrend = hdr.find(NULL_BYTE) typ, size = hdr[:hdrend].split(" ".encode("ascii")) size = int(size) self._s = size @@ -113,7 +117,7 @@ def _parse_header_info(self): self._phi = True - return typ.decode("ascii"), size + return force_text(typ), size #{ Interface @@ -264,10 +268,7 @@ def read(self, size=-1): # END adjust winsize # takes a slice, but doesn't copy the data, it says ... - try: - indata = memoryview(self._m)[self._cws:self._cwe].tobytes() - except (NameError, TypeError): - indata = buffer(self._m, self._cws, self._cwe - self._cws) + indata = buffer(self._m, self._cws, self._cwe - self._cws) # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) @@ -379,6 +380,7 @@ def _set_cache_too_slow_without_c(self, attr): def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" + from gitdb.utils.compat import buffer # TODO: There should be a special case if there is only one stream # Then the default-git algorithm should perform a tad faster, as the @@ -546,7 +548,10 @@ def __init__(self): def write(self, data): """:raise IOError: If not all bytes could be written :return: length of incoming data""" + + data = force_bytes(data) self.sha1.update(data) + return len(data) # END stream interface @@ -589,8 +594,11 @@ def __getattr__(self, attr): return getattr(self.buf, attr) def write(self, data): + data = force_bytes(data) + alen = Sha1Writer.write(self, data) self.buf.write(self.zip.compress(data)) + return alen def close(self): @@ -630,11 +638,15 @@ def __init__(self, fd): def write(self, data): """:raise IOError: If not all bytes could be written :return: lenght of incoming data""" + data = force_bytes(data) + self.sha1.update(data) cdata = self.zip.compress(data) bytes_written = write(self.fd, cdata) + if bytes_written != len(cdata): raise self.exc + return len(data) def close(self): diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index a6cdbbe65..d0028b897 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -35,7 +35,7 @@ class TestDBBase(TestBase): """Base class providing testing routines on databases""" # data - two_lines = "1234\nhello world".encode("ascii") + two_lines = "1234\nhello world" all_data = (two_lines, ) def _assert_object_writing_simple(self, db): @@ -81,8 +81,7 @@ def _assert_object_writing(self, db): prev_ostream = db.set_ostream(ostream) assert type(prev_ostream) in ostreams or prev_ostream in ostreams - - istream = IStream(str_blob_type, len(data), BytesIO(data)) + istream = IStream(str_blob_type, len(data), BytesIO(data.encode("ascii"))) # store returns same istream instance, with new sha set my_istream = db.store(istream) @@ -131,8 +130,7 @@ def _assert_object_writing_async(self, db): """Test generic object writing using asynchronous access""" ni = 5000 def istream_generator(offset=0, ni=ni): - for data_src in xrange(ni): - print(type(data_src), type(offset)) + for data_src in range(ni): data = bytes(data_src + offset) yield IStream(str_blob_type, len(data), BytesIO(data)) # END for each item diff --git a/gitdb/util.py b/gitdb/util.py index 30c7008f4..5a82c553e 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -30,10 +30,7 @@ mman = SlidingWindowMapManager() #END handle mman -try: - import hashlib -except ImportError: - import sha +import hashlib try: from struct import unpack_from @@ -84,9 +81,8 @@ def unpack_from(fmt, data, offset=0): close = os.close fsync = os.fsync -# constants -NULL_HEX_SHA = "0"*40 -NULL_BIN_SHA = "\0"*20 +# Backwards compatibility imports +from gitdb.const import NULL_BIN_SHA, NULL_HEX_SHA #} END Aliases diff --git a/gitdb/utils/__init__.py b/gitdb/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py new file mode 100644 index 000000000..b9da683fa --- /dev/null +++ b/gitdb/utils/compat.py @@ -0,0 +1,39 @@ +import sys + +PY3 = sys.version_info[0] == 3 + +try: + from itertools import izip +except ImportError: + izip = zip + +try: + # Python 2 + buffer = buffer + memoryview = buffer +except NameError: + # Python 3 has no `buffer`; only `memoryview` + def buffer(obj, offset, size=None): + if size is None: + return memoryview(obj)[offset:] + else: + return memoryview(obj[offset:offset+size]) + + memoryview = memoryview + +if PY3: + MAXSIZE = sys.maxsize +else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py new file mode 100644 index 000000000..12164e756 --- /dev/null +++ b/gitdb/utils/encoding.py @@ -0,0 +1,35 @@ +from gitdb.utils import compat + +if compat.PY3: + string_types = (str, ) + text_type = str +else: + string_types = (basestring, ) + text_type = unicode + +def force_bytes(data, encoding="utf-8"): + if isinstance(data, bytes): + return data + + if isinstance(data, compat.memoryview): + return bytes(data) + + if isinstance(data, string_types): + return data.encode(encoding) + + return data + +def force_text(data, encoding="utf-8"): + if isinstance(data, text_type): + return data + + if isinstance(data, string_types): + return data.decode(encoding) + + if not isinstance(data, bytes): + data = force_bytes(data, encoding) + + if compat.PY3: + return text_type(data, encoding) + else: + return text_type(data) From ecdae96cdb8bbde322f59fd119dab302e7797c18 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Wed, 16 Jul 2014 20:36:02 -0400 Subject: [PATCH 0249/1392] Fixed a few more encoding issues Bytes should always be returned from the streams, so the tests should be checking against byte strings instead of text strings. This also fixes the `sha_iter` as it relied on the Python 2 `iterkeys` which has been renamed to `keys` in Python 3. --- gitdb/db/loose.py | 3 ++- gitdb/db/mem.py | 5 ++++- gitdb/test/db/lib.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index f66d6275f..63f96352b 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -52,6 +52,7 @@ ) from gitdb.utils.compat import MAXSIZE +from gitdb.utils.encoding import force_bytes import tempfile import mmap @@ -116,7 +117,7 @@ def partial_to_complete_sha_hex(self, partial_hexsha): :raise BadObject: """ candidate = None for binsha in self.sha_iter(): - if bin_to_hex(binsha).startswith(partial_hexsha): + if bin_to_hex(binsha).startswith(force_bytes(partial_hexsha)): # it can't ever find the same object twice if candidate is not None: raise AmbiguousObjectName(partial_hexsha) diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index efd85af27..a22454685 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -86,7 +86,10 @@ def size(self): return len(self._cache) def sha_iter(self): - return self._cache.iterkeys() + try: + return self._cache.iterkeys() + except AttributeError: + return self._cache.keys() #{ Interface diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index d0028b897..38747deb2 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -21,6 +21,7 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type +from gitdb.utils.encoding import force_bytes from async import IteratorReader @@ -97,7 +98,7 @@ def _assert_object_writing(self, db): assert info.size == len(data) ostream = db.stream(sha) - assert ostream.read() == data + assert ostream.read() == force_bytes(data) assert ostream.type == str_blob_type assert ostream.size == len(data) else: From c63dcacbfa996b1d0d81d50c359fa37e4906cfb1 Mon Sep 17 00:00:00 2001 From: Kevin Brown Date: Mon, 21 Jul 2014 20:38:17 -0400 Subject: [PATCH 0250/1392] Convert types to bytes This makes it easier to deal with things internally as now everything is passed as bytes. --- gitdb/fun.py | 32 +++++++++++++++++++------------- gitdb/stream.py | 4 ++-- gitdb/typ.py | 12 +++++------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index 2749f37d0..6495359bc 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -17,6 +17,12 @@ from itertools import islice from gitdb.utils.compat import izip +from gitdb.typ import ( + str_blob_type, + str_commit_type, + str_tree_type, + str_tag_type, +) from io import StringIO @@ -27,23 +33,23 @@ type_id_to_type_map = { 0 : "", # EXT 1 - 1 : "commit", - 2 : "tree", - 3 : "blob", - 4 : "tag", + 1 : str_commit_type, + 2 : str_tree_type, + 3 : str_blob_type, + 4 : str_tag_type, 5 : "", # EXT 2 OFS_DELTA : "OFS_DELTA", # OFFSET DELTA REF_DELTA : "REF_DELTA" # REFERENCE DELTA } -type_to_type_id_map = dict( - commit=1, - tree=2, - blob=3, - tag=4, - OFS_DELTA=OFS_DELTA, - REF_DELTA=REF_DELTA -) +type_to_type_id_map = { + str_commit_type: 1, + str_tree_type: 2, + str_blob_type: 3, + str_tag_type: 4, + "OFS_DELTA": OFS_DELTA, + "REF_DELTA": REF_DELTA, +} # used when dealing with larger streams chunk_size = 1000 * mmap.PAGESIZE @@ -398,7 +404,7 @@ def loose_object_header_info(m): hdr = decompressobj().decompress(m, decompress_size) type_name, size = hdr[:hdr.find(NULL_BYTE)].split(" ".encode("ascii")) - return force_text(type_name), int(size) + return type_name, int(size) def pack_object_header_info(data): """ diff --git a/gitdb/stream.py b/gitdb/stream.py index 9e49f50e0..5dd38ec97 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -29,7 +29,7 @@ from gitdb.const import NULL_BYTE from gitdb.utils.compat import buffer -from gitdb.utils.encoding import force_bytes, force_text +from gitdb.utils.encoding import force_bytes has_perf_mod = False try: @@ -117,7 +117,7 @@ def _parse_header_info(self): self._phi = True - return force_text(typ), size + return typ, size #{ Interface diff --git a/gitdb/typ.py b/gitdb/typ.py index e84dd2455..edd1f2731 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -4,11 +4,9 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing information about types known to the database""" -#{ String types +from gitdb.utils.encoding import force_bytes -str_blob_type = "blob" -str_commit_type = "commit" -str_tree_type = "tree" -str_tag_type = "tag" - -#} END string types +str_blob_type = force_bytes("blob") +str_commit_type = force_bytes("commit") +str_tree_type = force_bytes("tree") +str_tag_type = force_bytes("tag") From fda32852a1dd6f6b0988248fb2c7921104e3ea6c Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Fri, 25 Jul 2014 21:12:33 +0200 Subject: [PATCH 0251/1392] warnings fixes --- setup.py | 2 ++ smmap/test/test_buf.py | 44 ++++++++++++----------- smmap/test/test_mman.py | 79 ++++++++++++++++++++++------------------- 3 files changed, 68 insertions(+), 57 deletions(-) diff --git a/setup.py b/setup.py index 13d2063ab..204a2a75d 100644 --- a/setup.py +++ b/setup.py @@ -52,4 +52,6 @@ "Programming Language :: Python :: 3.4", ], long_description=long_description, + tests_require=('nose', 'nosexcover'), + test_suite='nose.collector' ) diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 15dfb8238..807d2770f 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,3 +1,5 @@ +from __future__ import with_statement, print_function + from .lib import TestBase, FileCreator from smmap.mman import SlidingWindowMapManager, StaticWindowMapManager @@ -17,65 +19,66 @@ static_man = StaticWindowMapManager() class TestBuf(TestBase): - + def test_basics(self): fc = FileCreator(self.k_window_test_size, "buffer_test") - + # invalid paths fail upon construction c = man_optimal.make_cursor(fc.path) self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large - + buf = SlidingWindowMapBuffer() # can create uninitailized buffers assert buf.cursor() is None - + # can call end access any time buf.end_access() buf.end_access() assert len(buf) == 0 - + # begin access can revive it, if the offset is suitable offset = 100 assert buf.begin_access(c, fc.size) == False assert buf.begin_access(c, offset) == True assert len(buf) == fc.size - offset assert buf.cursor().is_valid() - + # empty begin access keeps it valid on the same path, but alters the offset assert buf.begin_access() == True assert len(buf) == fc.size assert buf.cursor().is_valid() - + # simple access - data = open(fc.path, 'rb').read() + with open(fc.path, 'rb') as fp: + data = fp.read() assert data[offset] == buf[0] assert data[offset:offset*2] == buf[0:offset] - + # negative indices, partial slices assert buf[-1] == buf[len(buf)-1] assert buf[-10:] == buf[len(buf)-10:len(buf)] - + # end access makes its cursor invalid buf.end_access() assert not buf.cursor().is_valid() assert buf.cursor().is_associated() # but it remains associated - + # an empty begin access fixes it up again assert buf.begin_access() == True and buf.cursor().is_valid() del(buf) # ends access automatically del(c) - + assert man_optimal.num_file_handles() == 1 - + # PERFORMANCE - # blast away with rnadom access and a full mapping - we don't want to + # blast away with rnadom access and a full mapping - we don't want to # exagerate the manager's overhead, but measure the buffer overhead - # We do it once with an optimal setting, and with a worse manager which + # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! max_num_accesses = 100 fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): - for manager, man_id in ( (man_optimal, 'optimal'), + for manager, man_id in ( (man_optimal, 'optimal'), (man_worst_case, 'worst case'), (static_man, 'static optimal')): buf = SlidingWindowMapBuffer(manager.make_cursor(item)) @@ -84,7 +87,7 @@ def test_basics(self): num_accesses_left = max_num_accesses num_bytes = 0 fsize = fc.size - + st = time() buf.begin_access() while num_accesses_left: @@ -102,7 +105,7 @@ def test_basics(self): num_bytes += 1 #END handle mode # END handle num accesses - + buf.end_access() assert manager.num_file_handles() assert manager.collect() @@ -110,8 +113,9 @@ def test_basics(self): elapsed = max(time() - st, 0.001) # prevent zero division errors on windows mb = float(1000*1000) mode_str = (access_mode and "slice") or "single byte" - sys.stderr.write("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)\n" - % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed)) + print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" + % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed), + file=sys.stderr) # END handle access mode # END for each manager # END for each input diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index e0516b21c..4d1839eca 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,3 +1,5 @@ +from __future__ import with_statement, print_function + from .lib import TestBase, FileCreator from smmap.mman import * @@ -12,43 +14,43 @@ from copy import copy class TestMMan(TestBase): - + def test_cursor(self): fc = FileCreator(self.k_window_test_size, "cursor_test") - + man = SlidingWindowMapManager() ci = WindowCursor(man) # invalid cursor assert not ci.is_valid() assert not ci.is_associated() assert ci.size() == 0 # this is cached, so we can query it in invalid state - + cv = man.make_cursor(fc.path) assert not cv.is_valid() # no region mapped yet assert cv.is_associated()# but it know where to map it from assert cv.file_size() == fc.size assert cv.path() == fc.path - + # copy module cio = copy(cv) assert not cio.is_valid() and cio.is_associated() - + # assign method assert not ci.is_associated() ci.assign(cv) assert not ci.is_valid() and ci.is_associated() - + # unuse non-existing region is fine cv.unuse_region() cv.unuse_region() - + # destruction is fine (even multiple times) cv._destroy() WindowCursor(man)._destroy() - + def test_memory_manager(self): slide_man = SlidingWindowMapManager() static_man = StaticWindowMapManager() - + for man in (static_man, slide_man): assert man.num_file_handles() == 0 assert man.num_open_files() == 0 @@ -59,15 +61,15 @@ def test_memory_manager(self): assert man.window_size() > winsize_cmp_val assert man.mapped_memory_size() == 0 assert man.max_mapped_memory_size() > 0 - + # collection doesn't raise in 'any' mode man._collect_lru_region(0) # doesn't raise if we are within the limit man._collect_lru_region(10) - - # doesn't fail if we overallocate + + # doesn't fail if we overallocate assert man._collect_lru_region(sys.maxsize) == 0 - + # use a region, verify most basic functionality fc = FileCreator(self.k_window_test_size, "manager_test") fd = os.open(fc.path, os.O_RDONLY) @@ -77,8 +79,9 @@ def test_memory_manager(self): assert c.use_region(10, 10).is_valid() assert c.ofs_begin() == 10 assert c.size() == 10 - assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:] - + with open(fc.path, 'rb') as fp: + assert c.buffer()[:] == fp.read(20)[10:] + if isinstance(item, int): self.assertRaises(ValueError, c.path) else: @@ -87,38 +90,39 @@ def test_memory_manager(self): #END for each input os.close(fd) # END for each manager type - + def test_memman_operation(self): # test more access, force it to actually unmap regions fc = FileCreator(self.k_window_test_size, "manager_operation_test") - data = open(fc.path, 'rb').read() + with open(fc.path, 'rb') as fp: + data = fp.read() fd = os.open(fc.path, os.O_RDONLY) max_num_handles = 15 - #small_size = + #small_size = for mtype, args in ( (StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): for item in (fc.path, fd): assert len(data) == fc.size - + # small windows, a reasonable max memory. Not too many regions at once man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) c = man.make_cursor(item) - + # still empty (more about that is tested in test_memory_manager() assert man.num_open_files() == 0 assert man.mapped_memory_size() == 0 - + base_offset = 5000 # window size is 0 for static managers, hence size will be 0. We take that into consideration size = man.window_size() // 2 assert c.use_region(base_offset, size).is_valid() rr = c.region_ref() assert rr().client_count() == 2 # the manager and the cursor and us - + assert man.num_open_files() == 1 assert man.num_file_handles() == 1 assert man.mapped_memory_size() == rr().size() - + #assert c.size() == size # the cursor may overallocate in its static version assert c.ofs_begin() == base_offset assert rr().ofs_begin() == 0 # it was aligned and expanded @@ -127,9 +131,9 @@ def test_memman_operation(self): else: assert rr().size() == fc.size #END ignore static managers which dont use windows and are aligned to file boundaries - - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] - + + assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + # obtain second window, which spans the first part of the file - it is a still the same window nsize = (size or fc.size) - 10 assert c.use_region(0, nsize).is_valid() @@ -138,7 +142,7 @@ def test_memman_operation(self): assert c.size() == nsize assert c.ofs_begin() == 0 assert c.buffer()[:] == data[:nsize] - + # map some part at the end, our requested size cannot be kept overshoot = 4000 base_offset = fc.size - (size or c.size()) + overshoot @@ -156,23 +160,23 @@ def test_memman_operation(self): assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left assert rr().ofs_end() <= fc.size # it cannot be larger than the file assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] - + # unising a region makes the cursor invalid c.unuse_region() assert not c.is_valid() if man.window_size(): - # but doesn't change anything regarding the handle count - we cache it and only + # but doesn't change anything regarding the handle count - we cache it and only # remove mapped regions if we have to assert man.num_file_handles() == 2 #END ignore this for static managers - + # iterate through the windows, verify data contents # this will trigger map collection after a while max_random_accesses = 5000 num_random_accesses = max_random_accesses memory_read = 0 st = time() - + # cache everything to get some more performance includes_ofs = c.includes_ofs max_mapped_memory_size = man.max_mapped_memory_size() @@ -182,7 +186,7 @@ def test_memman_operation(self): while num_random_accesses: num_random_accesses -= 1 base_offset = randint(0, fc.size - 1) - + # precondition if man.window_size(): assert max_mapped_memory_size >= mapped_memory_size() @@ -192,19 +196,20 @@ def test_memman_operation(self): csize = c.size() assert c.buffer()[:] == data[base_offset:base_offset+csize] memory_read += csize - + assert includes_ofs(base_offset) assert includes_ofs(base_offset+csize-1) assert not includes_ofs(base_offset+csize) # END while we should do an access elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows mb = float(1000 * 1000) - sys.stderr.write("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed)) - + print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed), + file=sys.stderr) + # an offset as large as the size doesn't work ! assert not c.use_region(fc.size, size).is_valid() - + # collection - it should be able to collect all assert man.num_file_handles() assert man.collect() From cc32b0a99744b07bcacf3b027af597a4bdd10b85 Mon Sep 17 00:00:00 2001 From: Orgad Shaneh Date: Wed, 10 Sep 2014 09:51:54 +0300 Subject: [PATCH 0252/1392] Setup: Fix invalid syntax --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 62bc6d007..07f810be8 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def run(self): try: build_ext.run(self) except Exception: - print "Ignored failure when building extensions, pure python modules will be used instead" + print("Ignored failure when building extensions, pure python modules will be used instead") # END ignore errors From a9f7f7bc5a2819924992dcb536c8e72399ae6f16 Mon Sep 17 00:00:00 2001 From: Matt Hickford Date: Mon, 6 Oct 2014 17:35:19 +0100 Subject: [PATCH 0253/1392] Add installation instructions to readme Also, fix broken badge link. --- README.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 0fc0534e6..f97d5c31e 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,20 @@ aims at allowing full access to loose objects as well as packs with performance and scalability in mind. It operates exclusively on streams, allowing to operate on large objects with a small memory footprint. +Installation +============ + +.. image:: https://pypip.in/version/gitdb/badge.svg + :target: https://pypi.python.org/pypi/gitdb/ + :alt: Latest Version +.. image:: https://pypip.in/py_versions/gitdb/badge.svg + :target: https://pypi.python.org/pypi/gitdb/ + :alt: Supported Python versions + +From `PyPI `_ + + pip install gitdb + REQUIREMENTS ============ @@ -33,8 +47,9 @@ http://groups.google.com/group/git-python ISSUE TRACKER ============= -.. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master :target: https://travis-ci.org/gitpython-developers/gitdb - +.. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master + :target: https://travis-ci.org/gitpython-developers/gitdb + https://github.com/gitpython-developers/gitdb/issues LICENSE From 0e0b7e253f7635af9e24a15f4393481495565667 Mon Sep 17 00:00:00 2001 From: Matt Hickford Date: Mon, 6 Oct 2014 17:39:18 +0100 Subject: [PATCH 0254/1392] Clarify which Python versions are supported --- setup.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 07f810be8..6dd4fa4b4 100755 --- a/setup.py +++ b/setup.py @@ -90,5 +90,13 @@ def get_data_files(self): zip_safe=False, requires=('async (>=0.6.1)', 'smmap (>=0.8.0)'), install_requires=('async >= 0.6.1', 'smmap >= 0.8.0'), - long_description = """GitDB is a pure-Python git object database""" + long_description = """GitDB is a pure-Python git object database""", + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], ) From ef59e98f74eeb00373b02f7a2b8bed1943924de9 Mon Sep 17 00:00:00 2001 From: David Schryer Date: Wed, 5 Nov 2014 10:48:33 +0200 Subject: [PATCH 0255/1392] Minor change to begin Python3 compatibility support. --- gitdb/db/ref.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 60004a77a..0a28b9efb 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -68,7 +68,7 @@ def _update_dbs_from_ref_file(self): db.databases() # END verification self._dbs.append(db) - except Exception, e: + except Exception as e: # ignore invalid paths or issues pass # END for each path to add From 977e666e2489ddc669a06481bb5192b59854da8d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 12 Nov 2014 20:30:47 +0100 Subject: [PATCH 0256/1392] Initial improvements to get rid of the performance regression in py3. Byte buffer concatenations are considerably slower here for some reason. Also there was no need for the memorybuffer. --- .travis.yml | 2 ++ README.md | 7 +------ doc/source/changes.rst | 6 +++++- smmap/buf.py | 10 +++++----- smmap/mman.py | 29 ++++++++++++++--------------- smmap/util.py | 5 +++-- 6 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index 47cb41170..1f7872c86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ language: python python: + - 2.4 + - 2.5 - 2.6 - 2.7 - 3.3 diff --git a/README.md b/README.md index c056ed1c1..6d2353820 100644 --- a/README.md +++ b/README.md @@ -25,19 +25,15 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 2.4, 2.5 or 2.6 +* Python 2.4, 2.5, 2.6, 2.7 or 3.3 * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. - - ## Limitations * The memory access is read-only by design. * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. -* It wasn't tested on python 2.7 and 3.x. - ## Installing smmap @@ -80,7 +76,6 @@ Issues can be filed on github: * https://github.com/Byron/smmap/issues - ## License Information *smmap* is licensed under the New BSD License. diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 03148fb31..6174008e9 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,12 +2,16 @@ Changelog ######### +********** +v0.8.2 +********** +- Cleaned up code and assured it works sufficiently well with python 3 + ********** v0.8.1 ********** - A single bugfix - ********** v0.8.0 ********** diff --git a/smmap/buf.py b/smmap/buf.py index 2f27d4d01..4cde6dde4 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -79,18 +79,18 @@ def __getslice__(self, i, j): else: l = j-i # total length ofs = i - # Keeping tokens in a list could possible be faster, but the list - # overhead outweighs the benefits (tested) ! - md = bytes() + # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower + # in the previous iteration of this code + md = list() while l: c.use_region(ofs, l) assert c.is_valid() d = c.buffer()[:l] ofs += len(d) l -= len(d) - md += d + md.append(d) #END while there are bytes to read - return md + return bytes().join(md) # END fast or slow path #{ Interface diff --git a/smmap/mman.py b/smmap/mman.py index 7cbb535bd..a89efd474 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,13 +1,12 @@ """Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" from .util import ( - MapWindow, - MapRegion, - MapRegionList, - is_64_bit, - align_to_mmap, - string_types, - buffer, - ) + MapWindow, + MapRegion, + MapRegionList, + is_64_bit, + string_types, + buffer, + ) from weakref import ref import sys @@ -261,14 +260,14 @@ class StaticWindowMapManager(object): def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxsize): """initialize the manager with the given parameters. :param window_size: if -1, a default window size will be chosen depending on - the operating system's architechture. It will internally be quantified to a multiple of the page size + the operating system's architecture. It will internally be quantified to a multiple of the page size If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. - If 0, a viable default iwll be set dependning on the system's architecture. + If 0, a viable default will be set depending on the system's architecture. It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate - :param max_open_handles: if not maxin, limit the amount of open file handles to the given number. - Otherwise the amount is only limited by the system iteself. If a system or soft limit is hit, - the manager will free as many handles as posisble""" + :param max_open_handles: if not maxint, limit the amount of open file handles to the given number. + Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, + the manager will free as many handles as possible""" self._fdict = dict() self._window_size = window_size self._max_memory_size = max_memory_size @@ -277,7 +276,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. self._handle_count = 0 if window_size < 0: - coeff = 32 + coeff = 64 if is_64_bit(): coeff = 1024 #END handle arch @@ -285,7 +284,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. # END handle max window size if max_memory_size == 0: - coeff = 512 + coeff = 1024 if is_64_bit(): coeff = 8192 #END handle arch diff --git a/smmap/util.py b/smmap/util.py index c37dfdd31..3396d90c0 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -23,8 +23,9 @@ except NameError: # Python 3 has no `buffer`; only `memoryview` def buffer(obj, offset, size): - return memoryview(obj[offset:offset+size]) - + # return memoryview(obj[offset:offset+size]) + # doing it directly is much faster ! + return obj[offset:offset+size] def string_types(): if sys.version_info[0] >= 3: From 948a9274527d14702875581d7115389cf9aa8244 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 08:21:09 +0100 Subject: [PATCH 0257/1392] Fixed a few typos and major linter errors --- .travis.yml | 5 +++-- README.md | 4 +--- doc/source/changes.rst | 2 +- doc/source/intro.rst | 8 ++------ setup.py | 10 +++++----- smmap/__init__.py | 2 +- smmap/buf.py | 2 -- smmap/mman.py | 16 ++++++---------- smmap/test/test_buf.py | 14 +++++++++----- smmap/test/test_mman.py | 12 +++++++----- smmap/test/test_util.py | 9 ++++++++- smmap/util.py | 4 ++-- 12 files changed, 45 insertions(+), 43 deletions(-) mode change 100644 => 100755 setup.py diff --git a/.travis.yml b/.travis.yml index 1f7872c86..cb0c16e44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,8 @@ language: python python: - - 2.4 - - 2.5 + # These versions are unsupported by travis, even though smmap claims to still support these outdated versions + # - 2.4 + # - 2.5 - 2.6 - 2.7 - 3.3 diff --git a/README.md b/README.md index 6d2353820..327d66372 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,9 @@ The package was tested on all of the previously mentioned configurations. ## Installing smmap -Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the [setuptools](http://peak.telecommunity.com/DevCenter/setuptools) or [pip](http://www.pip-installer.org/en/latest) respectively: +Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: ```bash -$ easy_install smmap -# or $ pip install smmap ``` diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 6174008e9..d5ed8e378 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -3,7 +3,7 @@ Changelog ######### ********** -v0.8.2 +v0.8.3 ********** - Cleaned up code and assured it works sufficiently well with python 3 diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 30bff0ded..ee3108a6a 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -22,7 +22,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 2.4, 2.5 or 2.6 +* Python 2.4, 2.5, 2.6, 2.7 or 3.3 * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. @@ -32,15 +32,12 @@ Limitations ########### * The memory access is read-only by design. * In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. -* It wasn't tested on python 2.7 and 3.x. ################ Installing smmap ################ -Its easiest to install smmap using the *easy_install* or *pip* program, which is part of the `setuptools`_ or `pip`_ respectively:: +Its easiest to install smmap using the *pip* program:: - $ easy_install smmap - # or $ pip install smmap As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. @@ -75,5 +72,4 @@ License Information ################### *smmap* is licensed under the New BSD License. -.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools .. _pip: http://www.pip-installer.org/en/latest/ diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 204a2a75d..c6afc8960 --- a/setup.py +++ b/setup.py @@ -10,10 +10,10 @@ import smmap -if os.path.exists("README.rst"): - long_description = codecs.open('README.rst', "r", "utf-8").read() +if os.path.exists("README.md"): + long_description = codecs.open('README.md', "r", "utf-8").read() else: - long_description = "See http://github.com/nvie/smmap/tree/master" + long_description = "See http://github.com/Byron/smmap" setup( name="smmap", @@ -32,8 +32,8 @@ #"Development Status :: 1 - Planning", #"Development Status :: 2 - Pre-Alpha", #"Development Status :: 3 - Alpha", - "Development Status :: 4 - Beta", - #"Development Status :: 5 - Production/Stable", + # "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", #"Development Status :: 6 - Mature", #"Development Status :: 7 - Inactive", "Environment :: Console", diff --git a/smmap/__init__.py b/smmap/__init__.py index 879ebea24..c494648d7 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 2) +version_info = (0, 8, 3) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience diff --git a/smmap/buf.py b/smmap/buf.py index 4cde6dde4..ef9d49e46 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -1,6 +1,4 @@ """Module with a simple buffer implementation using the memory manager""" -from .mman import WindowCursor - import sys __all__ = ["SlidingWindowMapBuffer"] diff --git a/smmap/mman.py b/smmap/mman.py index a89efd474..da6fd8153 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -101,7 +101,7 @@ def use_region(self, offset = 0, size = 0, flags = 0): :param flags: additional flags to be given to os.open in case a file handle is initially opened for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. - This is not the case if the mapping failed becaues we reached the end of the file + This is not the case if the mapping failed because we reached the end of the file **Note:**: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" @@ -137,7 +137,7 @@ def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region **Note:** the cursor unuses the region automatically upon destruction. It is recommended - to unuse the region once you are done reading from it in persistent cursors as it + to un-use the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None # note: should reset ofs and size, but we spare that for performance. Its not @@ -203,7 +203,7 @@ def file_size(self): return self._rlist.file_size() def path_or_fd(self): - """:return: path or file decriptor of the underlying mapped file""" + """:return: path or file descriptor of the underlying mapped file""" return self._rlist.path_or_fd() def path(self): @@ -237,12 +237,12 @@ class StaticWindowMapManager(object): These clients would have to use a SlidingWindowMapBuffer to hide this fact. This type will always use a maximum window size, and optimize certain methods to - acomodate this fact""" + accommodate this fact""" __slots__ = [ '_fdict', # mapping of path -> StorageHelper (of some kind '_window_size', # maximum size of a window - '_max_memory_size', # maximum amount ofmemory we may allocate + '_max_memory_size', # maximum amount of memory we may allocate '_max_handle_count', # maximum amount of handles to keep open '_memory_size', # currently allocated memory size '_handle_count', # amount of currently allocated file handles @@ -264,7 +264,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default will be set depending on the system's architecture. - It is a soft limit that is tried to be kept, but nothing bad happens if we have to overallocate + It is a soft limit that is tried to be kept, but nothing bad happens if we have to over-allocate :param max_open_handles: if not maxint, limit the amount of open file handles to the given number. Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, the manager will free as many handles as possible""" @@ -350,8 +350,6 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # As many more operations are likely to fail in that condition ( # like reading a file from disk, etc) we free up as much as possible # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway if is_recursive: # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it @@ -562,8 +560,6 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # As many more operations are likely to fail in that condition ( # like reading a file from disk, etc) we free up as much as possible # As this invalidates our insert position, we have to recurse here - # NOTE: The c++ version uses a linked list to curcumvent this, but - # using that in python is probably too slow anyway if is_recursive: # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 807d2770f..d3e51e2ee 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,14 +1,18 @@ -from __future__ import with_statement, print_function +from __future__ import print_function from .lib import TestBase, FileCreator -from smmap.mman import SlidingWindowMapManager, StaticWindowMapManager -from smmap.buf import * +from smmap.mman import ( + SlidingWindowMapManager, + StaticWindowMapManager + ) +from smmap.buf import SlidingWindowMapBuffer from random import randint from time import time import sys import os +import logging man_optimal = SlidingWindowMapManager() @@ -71,8 +75,8 @@ def test_basics(self): assert man_optimal.num_file_handles() == 1 # PERFORMANCE - # blast away with rnadom access and a full mapping - we don't want to - # exagerate the manager's overhead, but measure the buffer overhead + # blast away with random access and a full mapping - we don't want to + # exaggerate the manager's overhead, but measure the buffer overhead # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! max_num_accesses = 100 diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 4d1839eca..cc5d91488 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,11 +1,13 @@ -from __future__ import with_statement, print_function +from __future__ import print_function from .lib import TestBase, FileCreator -from smmap.mman import * -from smmap.mman import WindowCursor +from smmap.mman import ( + WindowCursor, + SlidingWindowMapManager, + StaticWindowMapManager + ) from smmap.util import align_to_mmap -from smmap.exc import RegionCollectionError from random import randint from time import time @@ -67,7 +69,7 @@ def test_memory_manager(self): # doesn't raise if we are within the limit man._collect_lru_region(10) - # doesn't fail if we overallocate + # doesn't fail if we over-allocate assert man._collect_lru_region(sys.maxsize) == 0 # use a region, verify most basic functionality diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 8afba005e..745da83d7 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,6 +1,13 @@ from .lib import TestBase, FileCreator -from smmap.util import * +from smmap.util import ( + MapWindow, + MapRegion, + MapRegionList, + ALLOCATIONGRANULARITY, + is_64_bit, + align_to_mmap + ) import os import sys diff --git a/smmap/util.py b/smmap/util.py index 3396d90c0..a4d7d8f1d 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -113,7 +113,7 @@ class MapRegion(object): '__weakref__' ] _need_compat_layer = sys.version_info[0] < 3 and sys.version_info[1] < 6 - + if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset #END handle additional slot @@ -283,4 +283,4 @@ def file_size(self): #END update file size return self._file_size -#} END utilty classes +#} END utility classes From 85dde34bc724570617f3df1cdc40ba1b0942c77e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 09:00:44 +0100 Subject: [PATCH 0258/1392] Minor adjustments to adapt to changes in async (due to be removed anyway) --- gitdb/__init__.py | 2 +- gitdb/base.py | 6 +----- gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- gitdb/fun.py | 2 +- gitdb/pack.py | 3 ++- gitdb/stream.py | 3 +-- gitdb/test/lib.py | 2 -- gitdb/test/test_stream.py | 3 +-- gitdb/util.py | 23 +++++++++-------------- setup.py | 35 ++++++++++++++++++++++++++--------- 11 files changed, 44 insertions(+), 39 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index ff750d14c..bb1d13b47 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -27,7 +27,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 4) +version_info = (0, 5, 5) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/base.py b/gitdb/base.py index bad5f7472..3476d0d1a 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -3,11 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with basic data structures - they are designed to be lightweight and fast""" -from util import ( - bin_to_hex, - zlib - ) - +from util import bin_to_hex from fun import ( type_id_to_type_map, type_to_type_id_map diff --git a/gitdb/ext/async b/gitdb/ext/async index 90326fb86..b930ee15c 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 90326fb867f94b193c277b07b23e364047e1ed28 +Subproject commit b930ee15c029860285db60aab4913dc8a9af2cd9 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 616e9ceaf..f53ddc686 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 616e9ceaf917e4d8f3cf2c145401b8069ce307dd +Subproject commit f53ddc686c0d226b2c69cc3732406dd3796932cf diff --git a/gitdb/fun.py b/gitdb/fun.py index c1e73e895..177a2fb56 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -10,7 +10,7 @@ BadObjectType ) -from util import zlib +import zlib decompressobj = zlib.decompressobj import mmap diff --git a/gitdb/pack.py b/gitdb/pack.py index 48121f026..e2673ee1b 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -3,13 +3,14 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains PackIndexFile and PackFile implementations""" +import zlib + from gitdb.exc import ( BadObject, UnsupportedOperation, ParseError ) from util import ( - zlib, mman, LazyMixin, unpack_from, diff --git a/gitdb/stream.py b/gitdb/stream.py index 6441b1e1a..cbb10c1f1 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -4,7 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from cStringIO import StringIO -import errno +import zlib import mmap import os @@ -23,7 +23,6 @@ make_sha, write, close, - zlib ) has_perf_mod = False diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index ac8473a4e..af57e46d1 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -11,8 +11,6 @@ ZippedStoreShaWriter ) -from gitdb.util import zlib - import sys import random from array import array diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 6dc27463c..d2487f6d1 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -18,12 +18,11 @@ hex_to_bin ) -from gitdb.util import zlib +import zlib from gitdb.typ import ( str_blob_type ) -import time import tempfile import os diff --git a/gitdb/util.py b/gitdb/util.py index 1662b662d..75f6bb927 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -7,27 +7,22 @@ import mmap import sys import errno - -from cStringIO import StringIO +import stat # in py 2.4, StringIO is only StringI, without write support. # Hence we must use the python implementation for this if sys.version_info[1] < 5: from StringIO import StringIO +else: + from cStringIO import StringIO # END handle python 2.4 -try: - import async.mod.zlib as zlib -except ImportError: - import zlib -# END try async zlib - from async import ThreadPool from smmap import ( - StaticWindowMapManager, - SlidingWindowMapManager, - SlidingWindowMapBuffer - ) + StaticWindowMapManager, + SlidingWindowMapManager, + SlidingWindowMapBuffer + ) # initialize our global memory manager instance # Use it to free cached (and unused) resources. @@ -304,7 +299,7 @@ def open(self, write=False, stream=False): binary = getattr(os, 'O_BINARY', 0) lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary try: - fd = os.open(self._lockfilepath(), lockmode, 0600) + fd = os.open(self._lockfilepath(), lockmode, stat.S_IREAD|stat.S_IWRITE) if not write: os.close(fd) else: @@ -373,7 +368,7 @@ def _end_writing(self, successful=True): # assure others can at least read the file - the tmpfile left it at rw-- # We may also write that file, on windows that boils down to a remove- # protection as well - chmod(self._filepath, 0644) + chmod(self._filepath, stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH) else: # just delete the file so far, we failed os.remove(lockfile) diff --git a/setup.py b/setup.py index 6dd4fa4b4..639370471 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 4) +version_info = (0, 5, 5) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext':build_ext_nofail}, @@ -88,15 +88,32 @@ def get_data_files(self): ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('async (>=0.6.1)', 'smmap (>=0.8.0)'), + requires=('async (>=0.6.2)', 'smmap (>=0.8.3)'), install_requires=('async >= 0.6.1', 'smmap >= 0.8.0'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - ], - ) + # Picked from + # http://pypi.python.org/pypi?:action=list_classifiers + #"Development Status :: 1 - Planning", + #"Development Status :: 2 - Pre-Alpha", + #"Development Status :: 3 - Alpha", + # "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", + #"Development Status :: 6 - Mature", + #"Development Status :: 7 - Inactive", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + ],) From a8f2f63823324ad76cbb36b0f4115e73c7d9d594 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 10:31:45 +0100 Subject: [PATCH 0259/1392] Made sure xrange is used instead of range in python 2 range in py2 will return a list, which can mean a lot of time and memory is spent on generating it even though it's just used for iteration. Simplified implementation of MAXSIZE --- gitdb/db/pack.py | 3 ++- gitdb/ext/async | 2 +- gitdb/ext/smmap | 2 +- gitdb/pack.py | 10 +++++----- gitdb/test/db/lib.py | 5 +++-- gitdb/test/lib.py | 3 ++- gitdb/test/test_pack.py | 3 ++- gitdb/util.py | 5 +---- gitdb/utils/compat.py | 20 +++++--------------- 9 files changed, 22 insertions(+), 31 deletions(-) diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index eca02bbff..b95bfed9d 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -18,6 +18,7 @@ ) from gitdb.pack import PackEntity +from gitdb.utils.compat import xrange from functools import reduce @@ -106,7 +107,7 @@ def sha_iter(self): for entity in self.entities(): index = entity.index() sha_by_index = index.sha - for index in range(index.size()): + for index in xrange(index.size()): yield sha_by_index(index) # END for each index # END for each entity diff --git a/gitdb/ext/async b/gitdb/ext/async index 3f26b05c2..b930ee15c 160000 --- a/gitdb/ext/async +++ b/gitdb/ext/async @@ -1 +1 @@ -Subproject commit 3f26b05c2f1a079d5807ed15c01b053ee846e745 +Subproject commit b930ee15c029860285db60aab4913dc8a9af2cd9 diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 552671191..f53ddc686 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 55267119140f3828a24b4986600ed21a1808d6cc +Subproject commit f53ddc686c0d226b2c69cc3732406dd3796932cf diff --git a/gitdb/pack.py b/gitdb/pack.py index 4e83ba39d..0b3ffd53e 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -64,7 +64,7 @@ from binascii import crc32 from gitdb.const import NULL_BYTE -from gitdb.utils.compat import izip, buffer +from gitdb.utils.compat import izip, buffer, xrange import tempfile import array @@ -208,7 +208,7 @@ def write(self, pack_sha, write): for t in self._objs: tmplist[ord(t[0][0])] += 1 #END prepare fanout - for i in range(255): + for i in xrange(255): v = tmplist[i] sha_write(pack('>L', v)) tmplist[i+1] += v @@ -374,7 +374,7 @@ def _read_fanout(self, byte_offset): d = self._cursor.map() out = list() append = out.append - for i in range(256): + for i in xrange(256): append(unpack_from('>L', d, byte_offset + i*4)[0]) # END for each entry return out @@ -415,7 +415,7 @@ def offsets(self): a.byteswap() return a else: - return tuple(self.offset(index) for index in range(self.size())) + return tuple(self.offset(index) for index in xrange(self.size())) # END handle version def sha_to_index(self, sha): @@ -703,7 +703,7 @@ def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" _sha = self._index.sha _object = self._object - for index in range(self._index.size()): + for index in xrange(self._index.size()): yield _object(_sha(index), as_stream, index) # END for each index diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 38747deb2..8e333ddf2 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -22,6 +22,7 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type from gitdb.utils.encoding import force_bytes +from gitdb.utils.compat import xrange from async import IteratorReader @@ -43,7 +44,7 @@ def _assert_object_writing_simple(self, db): # write a bunch of objects and query their streams and info null_objs = db.size() ni = 250 - for i in range(ni): + for i in xrange(ni): data = pack(">L", i) istream = IStream(str_blob_type, len(data), BytesIO(data)) new_istream = db.store(istream) @@ -131,7 +132,7 @@ def _assert_object_writing_async(self, db): """Test generic object writing using asynchronous access""" ni = 5000 def istream_generator(offset=0, ni=ni): - for data_src in range(ni): + for data_src in xrange(ni): data = bytes(data_src + offset) yield IStream(str_blob_type, len(data), BytesIO(data)) # END for each item diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 75342f173..d692224ff 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -12,6 +12,7 @@ ) from gitdb.util import zlib +from gitdb.utils.compat import xrange import sys import random @@ -110,7 +111,7 @@ def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" actual_size = size_in_bytes // 4 - producer = range(actual_size) + producer = xrange(actual_size) if randomize: producer = list(producer) random.shuffle(producer) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index bcda3cfb8..6b67ad887 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -26,6 +26,7 @@ from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha +from gitdb.utils.compat import xrange from itertools import chain try: @@ -64,7 +65,7 @@ def _assert_index_file(self, index, version, size): assert len(index.offsets()) == size # get all data of all objects - for oidx in range(index.size()): + for oidx in xrange(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) diff --git a/gitdb/util.py b/gitdb/util.py index 5a82c553e..a3c44d446 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -122,13 +122,10 @@ def byte_ord(b): #{ Routines -def make_sha(source=None): +def make_sha(source=''.encode("ascii")): """A python2.4 workaround for the sha/hashlib module fiasco **Note** From the dulwich project """ - if source is None: - source = "".encode("ascii") - try: return hashlib.sha1(source) except NameError: diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index b9da683fa..eec319f3d 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -4,8 +4,10 @@ try: from itertools import izip + xrange = xrange except ImportError: izip = zip + xrange = range try: # Python 2 @@ -21,19 +23,7 @@ def buffer(obj, offset, size=None): memoryview = memoryview -if PY3: +try: + MAXSIZE = sys.maxint +except AttributeError: MAXSIZE = sys.maxsize -else: - # It's possible to have sizeof(long) != sizeof(Py_ssize_t). - class X(object): - def __len__(self): - return 1 << 31 - try: - len(X()) - except OverflowError: - # 32-bit - MAXSIZE = int((1 << 31) - 1) - else: - # 64-bit - MAXSIZE = int((1 << 63) - 1) - del X From 28fd45e0a7018f166820a5e00fce2ccb05ebdb61 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 11:28:27 +0100 Subject: [PATCH 0260/1392] Fixed incorrect usage of memoryview. It's not getting faster though [ skip ci ] --- smmap/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/util.py b/smmap/util.py index a4d7d8f1d..44e94124d 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -23,7 +23,7 @@ except NameError: # Python 3 has no `buffer`; only `memoryview` def buffer(obj, offset, size): - # return memoryview(obj[offset:offset+size]) + # return memoryview(obj)[offset:offset+size] # doing it directly is much faster ! return obj[offset:offset+size] From b64c771bcb2ec336dd549cfe9d072340c886f3c9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 12:17:01 +0100 Subject: [PATCH 0261/1392] Fixed all applicable lint issues --- gitdb/db/base.py | 2 -- gitdb/db/git.py | 7 +------ gitdb/db/loose.py | 8 -------- gitdb/db/pack.py | 1 - gitdb/db/ref.py | 1 - gitdb/ext/smmap | 2 +- gitdb/fun.py | 11 +++-------- gitdb/pack.py | 7 ++----- gitdb/stream.py | 4 +--- gitdb/test/db/lib.py | 8 ++++++-- gitdb/test/db/test_git.py | 6 +++++- gitdb/test/db/test_loose.py | 5 ++++- gitdb/test/db/test_mem.py | 5 ++++- gitdb/test/db/test_pack.py | 11 +++++++---- gitdb/test/lib.py | 9 +-------- gitdb/test/performance/lib.py | 4 +--- gitdb/test/performance/test_pack.py | 2 +- gitdb/test/performance/test_pack_streaming.py | 1 - gitdb/test/performance/test_stream.py | 18 +++++++----------- gitdb/test/test_base.py | 10 +++++++++- gitdb/test/test_example.py | 5 ++++- gitdb/test/test_pack.py | 3 --- gitdb/test/test_stream.py | 11 ++++++----- gitdb/test/test_util.py | 1 - gitdb/util.py | 6 +++++- gitdb/utils/compat.py | 10 ++++++++-- 26 files changed, 76 insertions(+), 82 deletions(-) diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 53a94d249..c534705b7 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -22,7 +22,6 @@ from itertools import chain from functools import reduce -import sys __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') @@ -206,7 +205,6 @@ def _databases_recursive(database, output): """Fill output list with database from db, in order. Deals with Loose, Packed and compound databases.""" if isinstance(database, CompoundDB): - compounds = list() dbs = database.databases() output.extend(db for db in dbs if not isinstance(db, CompoundDB)) for cdb in (db for db in dbs if isinstance(db, CompoundDB)): diff --git a/gitdb/db/git.py b/gitdb/db/git.py index 5c74a2049..d22e3f1b9 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -12,12 +12,7 @@ from gitdb.db.pack import PackedDB from gitdb.db.ref import ReferenceDB -from gitdb.util import LazyMixin -from gitdb.exc import ( - InvalidDBRoot, - BadObject, - AmbiguousObjectName -) +from gitdb.exc import InvalidDBRoot import os diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 63f96352b..3abdaa96f 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -10,7 +10,6 @@ from gitdb.exc import ( - InvalidDBRoot, BadObject, AmbiguousObjectName ) @@ -55,8 +54,6 @@ from gitdb.utils.encoding import force_bytes import tempfile -import mmap -import sys import os @@ -149,11 +146,6 @@ def _map_loose_object(self, sha): raise BadObject(sha) # END handle error # END exception handling - try: - return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - finally: - os.close(fd) - # END assure file is closed def set_ostream(self, stream): """:raise TypeError: if the stream does not support the Sha1Writer interface""" diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index b95bfed9d..4d0a7f8b3 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -103,7 +103,6 @@ def stream(self, sha): return entity.stream_at_index(index) def sha_iter(self): - sha_list = list() for entity in self.entities(): index = entity.index() sha_by_index = index.sha diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 748f7c145..d98912643 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -6,7 +6,6 @@ CompoundDB, ) -import os __all__ = ('ReferenceDB', ) class ReferenceDB(CompoundDB): diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index f53ddc686..28fd45e0a 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit f53ddc686c0d226b2c69cc3732406dd3796932cf +Subproject commit 28fd45e0a7018f166820a5e00fce2ccb05ebdb61 diff --git a/gitdb/fun.py b/gitdb/fun.py index 22e56ddbb..80f01e6b2 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -6,18 +6,15 @@ Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" -from gitdb.exc import ( - BadObjectType -) - import zlib from gitdb.util import byte_ord decompressobj = zlib.decompressobj import mmap from itertools import islice +from functools import reduce -from gitdb.utils.compat import izip +from gitdb.utils.compat import izip, buffer, xrange from gitdb.typ import ( str_blob_type, str_commit_type, @@ -247,7 +244,6 @@ def compress(self): if slen < 2: return self i = 0 - slen_orig = slen first_data_index = None while i < slen: @@ -399,7 +395,6 @@ def loose_object_header_info(m): object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" from gitdb.const import NULL_BYTE - from gitdb.utils.encoding import force_text decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) @@ -684,7 +679,7 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: - # raise ImportError; # DEBUG + # NOQA from _perf import connect_deltas except ImportError: pass diff --git a/gitdb/pack.py b/gitdb/pack.py index fbfd18b08..87818b6e7 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -7,6 +7,7 @@ from gitdb.exc import ( BadObject, + AmbiguousObjectName, UnsupportedOperation, ParseError ) @@ -57,11 +58,7 @@ FlexibleSha1Writer ) -from struct import ( - pack, - unpack, -) - +from struct import pack from binascii import crc32 from gitdb.const import NULL_BYTE diff --git a/gitdb/stream.py b/gitdb/stream.py index de5884859..5daf01cb6 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -3,9 +3,8 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from io import BytesIO, StringIO +from io import BytesIO -import errno import mmap import os import zlib @@ -15,7 +14,6 @@ stream_copy, apply_delta_data, connect_deltas, - DeltaChunkList, delta_types ) diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 8e333ddf2..67958c366 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -6,12 +6,16 @@ from gitdb.test.lib import ( with_rw_directory, with_packs_rw, - ZippedStoreShaWriter, fixture_path, TestBase ) -from gitdb.stream import Sha1Writer + + +from gitdb.stream import ( + Sha1Writer, + ZippedStoreShaWriter +) from gitdb.base import ( IStream, diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index cce2b9c09..4bce7dac8 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -2,7 +2,11 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + fixture_path, + with_rw_directory +) from gitdb.exc import BadObject from gitdb.db import GitDB from gitdb.base import OStream, OInfo diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 5e42b639a..1299f7b86 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -2,7 +2,10 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory +) from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index 9235b21d3..97f721719 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -2,7 +2,10 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory +) from gitdb.db import ( MemoryDB, LooseObjectDB diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index f5a4dcb92..1177cf962 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -2,9 +2,12 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory, + with_packs_rw +) from gitdb.db import PackedDB -from gitdb.test.lib import fixture_path from gitdb.exc import BadObject, AmbiguousObjectName @@ -46,8 +49,8 @@ def test_writing(self, path): random.shuffle(sha_list) for sha in sha_list: - info = pdb.info(sha) - stream = pdb.stream(sha) + pdb.info(sha) + pdb.stream(sha) # END for each sha to query diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index cb77e69bf..d88ec8b43 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -3,14 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" -from gitdb import ( - OStream, - ) -from gitdb.stream import ( - Sha1Writer, - ZippedStoreShaWriter -) - +from gitdb import OStream from gitdb.utils.compat import xrange import sys diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index 3563fcfbe..5b5c40e0d 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -4,9 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains library functions""" import os -from gitdb.test.lib import * -import shutil -import tempfile +from gitdb.test.lib import TestBase #{ Invvariants diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index 63856e218..b18e31ae6 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -9,11 +9,11 @@ from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB +from gitdb.utils.compat import xrange import sys import os from time import time -import random from nose import SkipTest diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index c66e60cba..297426303 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -38,7 +38,6 @@ def test_pack_writing(self): ni = 5000 count = 0 - total_size = 0 st = time() for sha in pdb.sha_iter(): count += 1 diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 010003d4b..6c8f71520 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -4,13 +4,13 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance data streaming performance""" from lib import TestBigRepoR -from gitdb.db import * -from gitdb.base import * -from gitdb.stream import * +from gitdb.db import LooseObjectDB +from gitdb.stream import IStream + from gitdb.util import ( - pool, - bin_to_hex - ) + pool, + bin_to_hex +) from gitdb.typ import str_blob_type from gitdb.fun import chunk_size @@ -19,19 +19,15 @@ ChannelThreadTask, ) -from cStringIO import StringIO from time import time import os import sys -import stat -import subprocess from lib import ( - TestBigRepoR, make_memory_file, with_rw_directory - ) +) #{ Utilities diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 4cca7dabe..578c29f73 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -9,7 +9,15 @@ DeriveTest, ) -from gitdb import * +from gitdb import ( + OInfo, + OPackInfo, + ODeltaPackInfo, + OStream, + OPackStream, + ODeltaPackStream, + IStream +) from gitdb.util import ( NULL_BIN_SHA ) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index c644b8849..433518c25 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -3,7 +3,10 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with examples from the tutorial section of the docs""" -from gitdb.test.lib import * +from gitdb.test.lib import ( + TestBase, + fixture_path +) from gitdb import IStream from gitdb.db import LooseObjectDB from gitdb.util import pool diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 6b67ad887..3ab2fec07 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -6,7 +6,6 @@ from gitdb.test.lib import ( TestBase, with_rw_directory, - with_packs_rw, fixture_path ) @@ -27,7 +26,6 @@ from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha from gitdb.utils.compat import xrange -from itertools import chain try: from itertools import izip @@ -37,7 +35,6 @@ from nose import SkipTest import os -import sys import tempfile diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 984d6e1d7..671a146da 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -7,17 +7,18 @@ from gitdb.test.lib import ( TestBase, DummyStream, - Sha1Writer, make_bytes, make_object, fixture_path ) -from gitdb import * -from gitdb.util import ( - NULL_HEX_SHA, - hex_to_bin +from gitdb import ( + DecompressMemMapReader, + FDCompressedSha1Writer, + LooseObjectDB, + Sha1Writer ) +from gitdb.util import hex_to_bin import zlib from gitdb.typ import ( diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index ec9a86ce7..e79355aaf 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -5,7 +5,6 @@ """Test for object db""" import tempfile import os -import sys from gitdb.test.lib import TestBase from gitdb.util import ( diff --git a/gitdb/util.py b/gitdb/util.py index 77a0023e6..8843e8c88 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -77,7 +77,10 @@ def unpack_from(fmt, data, offset=0): fsync = os.fsync # Backwards compatibility imports -from gitdb.const import NULL_BIN_SHA, NULL_HEX_SHA +from gitdb.const import ( + NULL_BIN_SHA, + NULL_HEX_SHA +) #} END Aliases @@ -124,6 +127,7 @@ def make_sha(source=''.encode("ascii")): try: return hashlib.sha1(source) except NameError: + import sha sha1 = sha.sha(source) return sha1 diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index eec319f3d..a2640fd23 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -6,8 +6,10 @@ from itertools import izip xrange = xrange except ImportError: + # py3 izip = zip xrange = range +# end handle python version try: # Python 2 @@ -15,11 +17,15 @@ memoryview = buffer except NameError: # Python 3 has no `buffer`; only `memoryview` + # However, it's faster to just slice the object directly, maybe it keeps a view internally def buffer(obj, offset, size=None): if size is None: - return memoryview(obj)[offset:] + # return memoryview(obj)[offset:] + return obj[offset:] else: - return memoryview(obj[offset:offset+size]) + # return memoryview(obj)[offset:offset+size] + return obj[offset:offset+size] + # end buffer reimplementation memoryview = memoryview From bf942a913d69eb2079f9e82888aaccf2f6222643 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 13:31:32 +0100 Subject: [PATCH 0262/1392] Fully removed all async dependencies --- .gitmodules | 3 - doc/source/changes.rst | 7 +++ gitdb/__init__.py | 4 +- gitdb/db/base.py | 55 ---------------- gitdb/db/mem.py | 8 +-- gitdb/db/pack.py | 4 -- gitdb/ext/async | 1 - gitdb/test/__init__.py | 12 ---- gitdb/test/db/lib.py | 83 ------------------------ gitdb/test/db/test_git.py | 1 - gitdb/test/db/test_loose.py | 1 - gitdb/test/performance/test_stream.py | 90 +-------------------------- gitdb/test/test_example.py | 23 ------- gitdb/util.py | 10 --- setup.py | 6 +- 15 files changed, 14 insertions(+), 294 deletions(-) delete mode 160000 gitdb/ext/async diff --git a/.gitmodules b/.gitmodules index 978105388..d85b15c9f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "async"] - path = gitdb/ext/async - url = https://github.com/gitpython-developers/async.git [submodule "smmap"] path = gitdb/ext/smmap url = https://github.com/Byron/smmap.git diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 839bf16a8..f544f76c0 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,13 @@ Changelog ######### +***** +0.6.0 +***** + +* Added support got python 3.X +* Removed all `async` dependencies and all `*_async` versions of methods with it. + ***** 0.5.4 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 72b5ab085..165993fc9 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -10,7 +10,7 @@ #{ Initialization def _init_externals(): """Initialize external projects by putting them into the path""" - for module in ('async', 'smmap'): + for module in ('smmap',): sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) try: @@ -27,7 +27,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 5) +version_info = (0, 6, 0) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/db/base.py b/gitdb/db/base.py index c534705b7..eac54ec38 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -4,7 +4,6 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains implementations of database retrieveing objects""" from gitdb.util import ( - pool, join, LazyMixin, hex_to_bin @@ -15,10 +14,6 @@ AmbiguousObjectName ) -from async import ( - ChannelThreadTask -) - from itertools import chain from functools import reduce @@ -41,47 +36,18 @@ def has_object(self, sha): binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") - def has_object_async(self, reader): - """Return a reader yielding information about the membership of objects - as identified by shas - :param reader: Reader yielding 20 byte shas. - :return: async.Reader yielding tuples of (sha, bool) pairs which indicate - whether the given sha exists in the database or not""" - task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha))) - return pool.add_task(task) - def info(self, sha): """ :return: OInfo instance :param sha: bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - def info_async(self, reader): - """Retrieve information of a multitude of objects asynchronously - :param reader: Channel yielding the sha's of the objects of interest - :return: async.Reader yielding OInfo|InvalidOInfo, in any order""" - task = ChannelThreadTask(reader, str(self.info_async), self.info) - return pool.add_task(task) - def stream(self, sha): """:return: OStream instance :param sha: 20 bytes binary sha :raise BadObject:""" raise NotImplementedError("To be implemented in subclass") - def stream_async(self, reader): - """Retrieve the OStream of multiple objects - :param reader: see ``info`` - :param max_threads: see ``ObjectDBW.store`` - :return: async.Reader yielding OStream|InvalidOStream instances in any order - - **Note:** depending on the system configuration, it might not be possible to - read all OStreams at once. Instead, read them individually using reader.read(x) - where x is small enough.""" - # base implementation just uses the stream method repeatedly - task = ChannelThreadTask(reader, str(self.stream_async), self.stream) - return pool.add_task(task) - def size(self): """:return: amount of objects in this database""" raise NotImplementedError() @@ -129,27 +95,6 @@ def store(self, istream): :raise IOError: if data could not be written""" raise NotImplementedError("To be implemented in subclass") - def store_async(self, reader): - """ - Create multiple new objects in the database asynchronously. The method will - return right away, returning an output channel which receives the results as - they are computed. - - :return: Channel yielding your IStream which served as input, in any order. - The IStreams sha will be set to the sha it received during the process, - or its error attribute will be set to the exception informing about the error. - - :param reader: async.Reader yielding IStream instances. - The same instances will be used in the output channel as were received - in by the Reader. - - **Note:** As some ODB implementations implement this operation atomic, they might - abort the whole operation if one item could not be processed. Hence check how - many items have actually been produced.""" - # base implementation uses store to perform the work - task = ChannelThreadTask(reader, str(self.store_async), self.store) - return pool.add_task(task) - #} END edit interface diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index a22454685..1aa0d511f 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -32,10 +32,7 @@ class MemoryDB(ObjectDBR, ObjectDBW): """A memory database stores everything to memory, providing fast IO and object retrieval. It should be used to buffer results and obtain SHAs before writing it to the actual physical storage, as it allows to query whether object already - exists in the target storage before introducing actual IO - - **Note:** memory is currently not threadsafe, hence the async methods cannot be used - for storing""" + exists in the target storage before introducing actual IO""" def __init__(self): super(MemoryDB, self).__init__() @@ -62,9 +59,6 @@ def store(self, istream): return istream - def store_async(self, reader): - raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access") - def has_object(self, sha): return sha in self._cache diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 4d0a7f8b3..eaf431a22 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -125,10 +125,6 @@ def store(self, istream): inefficient""" raise UnsupportedOperation() - def store_async(self, reader): - # TODO: add ObjectDBRW before implementing this - raise NotImplementedError() - #} END object db write diff --git a/gitdb/ext/async b/gitdb/ext/async deleted file mode 160000 index b930ee15c..000000000 --- a/gitdb/ext/async +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b930ee15c029860285db60aab4913dc8a9af2cd9 diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index ca104c0c5..8a681e428 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -2,15 +2,3 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php - -import gitdb.util - -#{ Initialization -def _init_pool(): - """Assure the pool is actually threaded""" - size = 2 - print("Setting ThreadPool to %i" % size) - gitdb.util.pool.set_size(size) - - -#} END initialization diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 67958c366..962d4bc2a 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -10,8 +10,6 @@ TestBase ) - - from gitdb.stream import ( Sha1Writer, ZippedStoreShaWriter @@ -28,8 +26,6 @@ from gitdb.utils.encoding import force_bytes from gitdb.utils.compat import xrange -from async import IteratorReader - from io import BytesIO from struct import pack @@ -132,82 +128,3 @@ def _assert_object_writing(self, db): # END for each data set # END for each dry_run mode - def _assert_object_writing_async(self, db): - """Test generic object writing using asynchronous access""" - ni = 5000 - def istream_generator(offset=0, ni=ni): - for data_src in xrange(ni): - data = bytes(data_src + offset) - yield IStream(str_blob_type, len(data), BytesIO(data)) - # END for each item - # END generator utility - - # for now, we are very trusty here as we expect it to work if it worked - # in the single-stream case - - # write objects - reader = IteratorReader(istream_generator()) - istream_reader = db.store_async(reader) - istreams = istream_reader.read() # read all - assert istream_reader.task().error() is None - assert len(istreams) == ni - - for stream in istreams: - assert stream.error is None - assert len(stream.binsha) == 20 - assert isinstance(stream, IStream) - # END assert each stream - - # test has-object-async - we must have all previously added ones - reader = IteratorReader( istream.binsha for istream in istreams ) - hasobject_reader = db.has_object_async(reader) - count = 0 - for sha, has_object in hasobject_reader: - assert has_object - count += 1 - # END for each sha - assert count == ni - - # read the objects we have just written - reader = IteratorReader( istream.binsha for istream in istreams ) - ostream_reader = db.stream_async(reader) - - # read items individually to prevent hitting possible sys-limits - count = 0 - for ostream in ostream_reader: - assert isinstance(ostream, OStream) - count += 1 - # END for each ostream - assert ostream_reader.task().error() is None - assert count == ni - - # get info about our items - reader = IteratorReader( istream.binsha for istream in istreams ) - info_reader = db.info_async(reader) - - count = 0 - for oinfo in info_reader: - assert isinstance(oinfo, OInfo) - count += 1 - # END for each oinfo instance - assert count == ni - - - # combined read-write using a converter - # add 2500 items, and obtain their output streams - nni = 2500 - reader = IteratorReader(istream_generator(offset=ni, ni=nni)) - istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ] - - istream_reader = db.store_async(reader) - istream_reader.set_post_cb(istream_to_sha) - - ostream_reader = db.stream_async(istream_reader) - - count = 0 - # read it individually, otherwise we might run into the ulimit - for ostream in ostream_reader: - assert isinstance(ostream, OStream) - count += 1 - # END for each ostream - assert count == nni diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 4bce7dac8..56899e579 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -48,4 +48,3 @@ def test_writing(self, path): # its possible to write objects self._assert_object_writing(gdb) - self._assert_object_writing_async(gdb) diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 1299f7b86..1d6af9c99 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -18,7 +18,6 @@ def test_basics(self, path): # write data self._assert_object_writing(ldb) - self._assert_object_writing_async(ldb) # verify sha iteration and size shas = list(ldb.sha_iter()) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 6c8f71520..929c7e537 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -7,18 +7,9 @@ from gitdb.db import LooseObjectDB from gitdb.stream import IStream -from gitdb.util import ( - pool, - bin_to_hex -) -from gitdb.typ import str_blob_type +from gitdb.util import bin_to_hex from gitdb.fun import chunk_size -from async import ( - IteratorReader, - ChannelThreadTask, - ) - from time import time import os import sys @@ -43,15 +34,6 @@ def read_chunked_stream(stream): return stream -class TestStreamReader(ChannelThreadTask): - """Expects input streams and reads them in chunks. It will read one at a time, - requireing a queue chunk of size 1""" - def __init__(self, *args): - super(TestStreamReader, self).__init__(*args) - self.fun = read_chunked_stream - self.max_chunksize = 1 - - #} END utilities class TestObjDBPerformance(TestBigRepoR): @@ -119,73 +101,3 @@ def test_large_data_streaming(self, path): # del db file so we keep something to do os.remove(db_file) # END for each randomization factor - - - # multi-threaded mode - # want two, should be supported by most of todays cpus - pool.set_size(2) - total_kib = 0 - nsios = len(string_ios) - for stream in string_ios: - stream.seek(0) - total_kib += len(stream.getvalue()) / 1000 - # END rewind - - def istream_iter(): - for stream in string_ios: - stream.seek(0) - yield IStream(str_blob_type, len(stream.getvalue()), stream) - # END for each stream - # END util - - # write multiple objects at once, involving concurrent compression - reader = IteratorReader(istream_iter()) - istream_reader = ldb.store_async(reader) - istream_reader.task().max_chunksize = 1 - - st = time() - istreams = istream_reader.read(nsios) - assert len(istreams) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Compressed %i KiB of data in loose odb in %f s ( %f Write KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - - # decompress multiple at once, by reading them - # chunk size is not important as the stream will not really be decompressed - - # until its read - istream_reader = IteratorReader(iter([ i.binsha for i in istreams ])) - ostream_reader = ldb.stream_async(istream_reader) - - chunk_task = TestStreamReader(ostream_reader, "chunker", None) - output_reader = pool.add_task(chunk_task) - output_reader.task().max_chunksize = 1 - - st = time() - assert len(output_reader.read(nsios)) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Decompressed %i KiB of data in loose odb in %f s ( %f Read KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) - - # store the files, and read them back. For the reading, we use a task - # as well which is chunked into one item per task. Reading all will - # very quickly result in two threads handling two bytestreams of - # chained compression/decompression streams - reader = IteratorReader(istream_iter()) - istream_reader = ldb.store_async(reader) - istream_reader.task().max_chunksize = 1 - - istream_to_sha = lambda items: [ i.binsha for i in items ] - istream_reader.set_post_cb(istream_to_sha) - - ostream_reader = ldb.stream_async(istream_reader) - - chunk_task = TestStreamReader(ostream_reader, "chunker", None) - output_reader = pool.add_task(chunk_task) - output_reader.max_chunksize = 1 - - st = time() - assert len(output_reader.read(nsios)) == nsios - elapsed = time() - st - - print >> sys.stderr, "Threads(%i): Compressed and decompressed and read %i KiB of data in loose odb in %f s ( %f Combined KiB / s)" % (pool.size(), total_kib, elapsed, total_kib / elapsed) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 433518c25..aa43a093f 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -9,12 +9,9 @@ ) from gitdb import IStream from gitdb.db import LooseObjectDB -from gitdb.util import pool from io import BytesIO -from async import IteratorReader - class TestExamples(TestBase): def test_base(self): @@ -45,23 +42,3 @@ def test_base(self): # now the sha is set assert len(istream.binsha) == 20 assert ldb.has_object(istream.binsha) - - - # async operation - # Create a reader from an iterator - reader = IteratorReader(ldb.sha_iter()) - - # get reader for object streams - info_reader = ldb.stream_async(reader) - - # read one - info = info_reader.read(1)[0] - - # read all the rest until depletion - ostreams = info_reader.read() - - # set the pool to use two threads - pool.set_size(2) - - # synchronize the mode of operation - pool.set_size(0) diff --git a/gitdb/util.py b/gitdb/util.py index 8843e8c88..93ba7f0ec 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -10,7 +10,6 @@ from io import StringIO -from async import ThreadPool from smmap import ( StaticWindowMapManager, SlidingWindowMapManager, @@ -43,15 +42,6 @@ def unpack_from(fmt, data, offset=0): # END own unpack_from implementation -#{ Globals - -# A pool distributing tasks, initially with zero threads, hence everything -# will be handled in the main thread -pool = ThreadPool(0) - -#} END globals - - #{ Aliases hex_to_bin = binascii.a2b_hex diff --git a/setup.py b/setup.py index 639370471..63ec5ddb3 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 5, 5) +version_info = (0, 6, 0) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext':build_ext_nofail}, @@ -88,8 +88,8 @@ def get_data_files(self): ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('async (>=0.6.2)', 'smmap (>=0.8.3)'), - install_requires=('async >= 0.6.1', 'smmap >= 0.8.0'), + requires=('smmap (>=0.8.3)'), + install_requires=('smmap >= 0.8.0'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ From 641b64c9f48139cf06774805d32892012fb9b82d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 18:31:17 +0100 Subject: [PATCH 0263/1392] Now tests work consistently in py2 and 3 It's a nice way of saying that there is still one failing, consistently. --- gitdb/const.py | 5 +- gitdb/db/base.py | 7 +- gitdb/db/loose.py | 2 +- gitdb/fun.py | 229 +++++++++++++++++++++++++------------ gitdb/pack.py | 21 ++-- gitdb/stream.py | 8 +- gitdb/test/db/test_pack.py | 2 +- gitdb/typ.py | 10 +- gitdb/utils/encoding.py | 4 +- 9 files changed, 178 insertions(+), 110 deletions(-) diff --git a/gitdb/const.py b/gitdb/const.py index 147f79cb9..6391d796f 100644 --- a/gitdb/const.py +++ b/gitdb/const.py @@ -1,5 +1,4 @@ -from gitdb.utils.encoding import force_bytes - -NULL_BYTE = force_bytes("\0") +BYTE_SPACE = b' ' +NULL_BYTE = b'\0' NULL_HEX_SHA = "0" * 40 NULL_BIN_SHA = NULL_BYTE * 20 diff --git a/gitdb/db/base.py b/gitdb/db/base.py index eac54ec38..a670eea63 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -9,6 +9,7 @@ hex_to_bin ) +from gitdb.utils.encoding import force_text from gitdb.exc import ( BadObject, AmbiguousObjectName @@ -122,8 +123,6 @@ def db_path(self, rela_path): """ :return: the given relative path relative to our database root, allowing to pontentially access datafiles""" - from gitdb.utils.encoding import force_text - return join(self._root_path, force_text(rela_path)) #} END interface @@ -234,12 +233,12 @@ def update_cache(self, force=False): def partial_to_complete_sha_hex(self, partial_hexsha): """ - :return: 20 byte binary sha1 from the given less-than-40 byte hexsha + :return: 20 byte binary sha1 from the given less-than-40 byte hexsha (bytes or str) :param partial_hexsha: hexsha with less than 40 byte :raise AmbiguousObjectName: """ databases = list() _databases_recursive(self, databases) - + partial_hexsha = force_text(partial_hexsha) len_partial_hexsha = len(partial_hexsha) if len_partial_hexsha % 2 != 0: partial_binsha = hex_to_bin(partial_hexsha + "0") diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 3abdaa96f..374302611 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -109,7 +109,7 @@ def readable_db_object_path(self, hexsha): def partial_to_complete_sha_hex(self, partial_hexsha): """:return: 20 byte binary sha1 string which matches the given name uniquely - :param name: hexadecimal partial name + :param name: hexadecimal partial name (bytes or ascii string) :raise AmbiguousObjectName: :raise BadObject: """ candidate = None diff --git a/gitdb/fun.py b/gitdb/fun.py index 80f01e6b2..064680adb 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -14,7 +14,9 @@ from itertools import islice from functools import reduce -from gitdb.utils.compat import izip, buffer, xrange +from gitdb.const import NULL_BYTE, BYTE_SPACE +from gitdb.utils.encoding import force_text +from gitdb.utils.compat import izip, buffer, xrange, PY3 from gitdb.typ import ( str_blob_type, str_commit_type, @@ -30,12 +32,12 @@ delta_types = (OFS_DELTA, REF_DELTA) type_id_to_type_map = { - 0 : "", # EXT 1 + 0 : b'', # EXT 1 1 : str_commit_type, 2 : str_tree_type, 3 : str_blob_type, 4 : str_tag_type, - 5 : "", # EXT 2 + 5 : b'', # EXT 2 OFS_DELTA : "OFS_DELTA", # OFFSET DELTA REF_DELTA : "REF_DELTA" # REFERENCE DELTA } @@ -394,11 +396,9 @@ def loose_object_header_info(m): :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the object as well as its uncompressed size in bytes. :param m: memory map from which to read the compressed object data""" - from gitdb.const import NULL_BYTE - decompress_size = 8192 # is used in cgit as well hdr = decompressobj().decompress(m, decompress_size) - type_name, size = hdr[:hdr.find(NULL_BYTE)].split(" ".encode("ascii")) + type_name, size = hdr[:hdr.find(NULL_BYTE)].split(BYTE_SPACE) return type_name, int(size) @@ -413,12 +413,21 @@ def pack_object_header_info(data): type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size s = 4 # starting bit-shift size - while c & 0x80: - c = byte_ord(data[i]) - i += 1 - size += (c & 0x7f) << s - s += 7 - # END character loop + if PY3: + while c & 0x80: + c = data[i] + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + else: + while c & 0x80: + c = ord(data[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop + # end performance at expense of maintenance ... return (type_id, size, i) def create_pack_object_header(obj_type, obj_size): @@ -429,16 +438,29 @@ def create_pack_object_header(obj_type, obj_size): :param obj_type: pack type_id of the object :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte - hdr = str() # output string - - c = (obj_type << 4) | (obj_size & 0xf) - obj_size >>= 4 - while obj_size: - hdr += chr(c | 0x80) - c = obj_size & 0x7f - obj_size >>= 7 - #END until size is consumed - hdr += chr(c) + if PY3: + hdr = bytearray() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr.append(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + #END until size is consumed + hdr.append(c) + else: + hdr = bytes() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr += chr(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + #END until size is consumed + hdr += chr(c) + # end handle interpreter return hdr def msb_size(data, offset=0): @@ -449,24 +471,36 @@ def msb_size(data, offset=0): i = 0 l = len(data) hit_msb = False - while i < l: - c = byte_ord(data[i+offset]) - size |= (c & 0x7f) << i*7 - i += 1 - if not c & 0x80: - hit_msb = True - break - # END check msb bit - # END while in range + if PY3: + while i < l: + c = data[i+offset] + size |= (c & 0x7f) << i*7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + else: + while i < l: + c = ord(data[i+offset]) + size |= (c & 0x7f) << i*7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range + # end performance ... if not hit_msb: raise AssertionError("Could not find terminating MSB byte in data stream") return i+offset, size def loose_object_header(type, size): """ - :return: string representing the loose object header, which is immediately + :return: bytes representing the loose object header, which is immediately followed by the content stream of size 'size'""" - return "%s %i\0" % (type, size) + return ('%s %i\0' % (force_text(type), size)).encode('ascii') def write_object(type, size, read, write, chunk_size=chunk_size): """ @@ -611,48 +645,93 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): **Note:** transcribed to python from the similar routine in patch-delta.c""" i = 0 db = delta_buf - while i < delta_buf_size: - c = ord(db[i]) - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = ord(db[i]) - i += 1 - if (c & 0x02): - cp_off |= (ord(db[i]) << 8) - i += 1 - if (c & 0x04): - cp_off |= (ord(db[i]) << 16) - i += 1 - if (c & 0x08): - cp_off |= (ord(db[i]) << 24) - i += 1 - if (c & 0x10): - cp_size = ord(db[i]) - i += 1 - if (c & 0x20): - cp_size |= (ord(db[i]) << 8) - i += 1 - if (c & 0x40): - cp_size |= (ord(db[i]) << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > src_buf_size): - break - write(buffer(src_buf, cp_off, cp_size)) - elif c: - write(db[i:i+c]) - i += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data + if PY3: + while i < delta_buf_size: + c = db[i] + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = db[i] + i += 1 + if (c & 0x02): + cp_off |= (db[i] << 8) + i += 1 + if (c & 0x04): + cp_off |= (db[i] << 16) + i += 1 + if (c & 0x08): + cp_off |= (db[i] << 24) + i += 1 + if (c & 0x10): + cp_size = db[i] + i += 1 + if (c & 0x20): + cp_size |= (db[i] << 8) + i += 1 + if (c & 0x40): + cp_size |= (db[i] << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(buffer(src_buf, cp_off, cp_size)) + elif c: + write(db[i:i+c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + else: + while i < delta_buf_size: + c = ord(db[i]) + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = ord(db[i]) + i += 1 + if (c & 0x02): + cp_off |= (ord(db[i]) << 8) + i += 1 + if (c & 0x04): + cp_off |= (ord(db[i]) << 16) + i += 1 + if (c & 0x08): + cp_off |= (ord(db[i]) << 24) + i += 1 + if (c & 0x10): + cp_size = ord(db[i]) + i += 1 + if (c & 0x20): + cp_size |= (ord(db[i]) << 8) + i += 1 + if (c & 0x40): + cp_size |= (ord(db[i]) << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(buffer(src_buf, cp_off, cp_size)) + elif c: + write(db[i:i+c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data + # end save byte_ord call and prevent performance regression in py2 # yes, lets use the exact same error message that git uses :) assert i == delta_buf_size, "delta replay has gone wild" diff --git a/gitdb/pack.py b/gitdb/pack.py index 87818b6e7..375cc59a9 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -86,8 +86,6 @@ def pack_object_at(cursor, offset, as_stream): :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" - from gitdb.utils.encoding import force_bytes - data = cursor.use_region(offset).buffer() type_id, uncomp_size, data_rela_offset = pack_object_header_info(data) total_rela_offset = None # set later, actual offset until data stream begins @@ -96,11 +94,11 @@ def pack_object_at(cursor, offset, as_stream): # OFFSET DELTA if type_id == OFS_DELTA: i = data_rela_offset - c = ord(data[i]) + c = byte_ord(data[i]) i += 1 delta_offset = c & 0x7f while c & 0x80: - c = ord(data[i]) + c = byte_ord(data[i]) i += 1 delta_offset += 1 delta_offset = (delta_offset << 7) + (c & 0x7f) @@ -118,12 +116,10 @@ def pack_object_at(cursor, offset, as_stream): # END handle type id abs_data_offset = offset + total_rela_offset if as_stream: - buff = buffer(data, total_rela_offset) - stream = DecompressMemMapReader(buff, False, uncomp_size) + stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: - delta_info = force_bytes(delta_info) return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream) else: if delta_info is None: @@ -204,7 +200,7 @@ def write(self, pack_sha, write): # fanout tmplist = list((0,)*256) # fanout or list with 64 bit offsets for t in self._objs: - tmplist[ord(t[0][0])] += 1 + tmplist[byte_ord(t[0][0])] += 1 #END prepare fanout for i in xrange(255): v = tmplist[i] @@ -215,7 +211,7 @@ def write(self, pack_sha, write): # sha1 ordered # save calls, that is push them into c - sha_write(''.join(t[0] for t in self._objs)) + sha_write(b''.join(t[0] for t in self._objs)) # crc32 for t in self._objs: @@ -258,7 +254,7 @@ class PackIndexFile(LazyMixin): # used in v2 indices _sha_list_offset = 8 + 1024 - index_v2_signature = '\377tOc' + index_v2_signature = b'\xfftOc' index_version_default = 2 def __init__(self, indexpath): @@ -446,13 +442,14 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): """ :return: index as in `sha_to_index` or None if the sha was not found in this index file - :param partial_bin_sha: an at least two bytes of a partial binary sha + :param partial_bin_sha: an at least two bytes of a partial binary sha as bytes :param canonical_length: lenght of the original hexadecimal representation of the given partial binary sha :raise AmbiguousObjectName:""" if len(partial_bin_sha) < 2: raise ValueError("Require at least 2 bytes of partial sha") + assert isinstance(partial_bin_sha, bytes), "partial_bin_sha must be bytes" first_byte = byte_ord(partial_bin_sha[0]) get_sha = self.sha @@ -680,7 +677,7 @@ def _set_cache_(self, attr): else: iter_offsets = iter(offsets_sorted) iter_offsets_plus_one = iter(offsets_sorted) - iter_offsets_plus_one.next() + next(iter_offsets_plus_one) consecutive = izip(iter_offsets, iter_offsets_plus_one) offset_map = dict(consecutive) diff --git a/gitdb/stream.py b/gitdb/stream.py index 5daf01cb6..43aa8e368 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -25,7 +25,7 @@ close, ) -from gitdb.const import NULL_BYTE +from gitdb.const import NULL_BYTE, BYTE_SPACE from gitdb.utils.compat import buffer from gitdb.utils.encoding import force_bytes @@ -102,7 +102,7 @@ def _parse_header_info(self): self._s = maxb hdr = self.read(maxb) hdrend = hdr.find(NULL_BYTE) - typ, size = hdr[:hdrend].split(" ".encode("ascii")) + typ, size = hdr[:hdrend].split(BYTE_SPACE) size = int(size) self._s = size @@ -378,8 +378,6 @@ def _set_cache_too_slow_without_c(self, attr): def _set_cache_brute_(self, attr): """If we are here, we apply the actual deltas""" - from gitdb.utils.compat import buffer - # TODO: There should be a special case if there is only one stream # Then the default-git algorithm should perform a tad faster, as the # delta is not peaked into, causing less overhead. @@ -421,7 +419,7 @@ def _set_cache_brute_(self, attr): # For the actual copying, we use a seek and write pattern of buffer # slices. final_target_size = None - for (dbuf, offset, src_size, target_size), dstream in reversed(zip(buffer_info_list, self._dstreams)): + for (dbuf, offset, src_size, target_size), dstream in zip(reversed(buffer_info_list), reversed(self._dstreams)): # allocate a buffer to hold all delta data - fill in the data for # fast access. We do this as we know that reading individual bytes # from our stream would be slower than necessary ( although possible ) diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 1177cf962..963a71af7 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -73,4 +73,4 @@ def test_writing(self, path): # assert num_ambiguous # non-existing - self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, "\0\0", 4) + self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, b'\0\0', 4) diff --git a/gitdb/typ.py b/gitdb/typ.py index edd1f2731..bc7ba5828 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -4,9 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing information about types known to the database""" -from gitdb.utils.encoding import force_bytes - -str_blob_type = force_bytes("blob") -str_commit_type = force_bytes("commit") -str_tree_type = force_bytes("tree") -str_tag_type = force_bytes("tag") +str_blob_type = b'blob' +str_commit_type = b'commit' +str_tree_type = b'tree' +str_tag_type = b'tag' diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 12164e756..617b51c83 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -11,9 +11,6 @@ def force_bytes(data, encoding="utf-8"): if isinstance(data, bytes): return data - if isinstance(data, compat.memoryview): - return bytes(data) - if isinstance(data, string_types): return data.encode(encoding) @@ -27,6 +24,7 @@ def force_text(data, encoding="utf-8"): return data.decode(encoding) if not isinstance(data, bytes): + assert False, "Shouldn't be here" data = force_bytes(data, encoding) if compat.PY3: From 8ae4e9579a263684c6b760aec2869be480ff22ba Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 18:46:49 +0100 Subject: [PATCH 0264/1392] reduced usage of force_bytes as clients are expected to pass bytes. It was useful for debugging though, maybe an explicit type assertions would help others ? As 'others' will be gitpython, I suppose I can handle it myself --- gitdb/stream.py | 6 +----- gitdb/test/db/lib.py | 7 +++---- gitdb/test/db/test_ref.py | 8 +++++--- gitdb/utils/encoding.py | 6 +----- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index 43aa8e368..e32fcf380 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -543,9 +543,9 @@ def __init__(self): def write(self, data): """:raise IOError: If not all bytes could be written + :param data: byte object :return: length of incoming data""" - data = force_bytes(data) self.sha1.update(data) return len(data) @@ -590,8 +590,6 @@ def __getattr__(self, attr): return getattr(self.buf, attr) def write(self, data): - data = force_bytes(data) - alen = Sha1Writer.write(self, data) self.buf.write(self.zip.compress(data)) @@ -634,8 +632,6 @@ def __init__(self, fd): def write(self, data): """:raise IOError: If not all bytes could be written :return: lenght of incoming data""" - data = force_bytes(data) - self.sha1.update(data) cdata = self.zip.compress(data) bytes_written = write(self.fd, cdata) diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 962d4bc2a..af6d9e0fd 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -23,7 +23,6 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type -from gitdb.utils.encoding import force_bytes from gitdb.utils.compat import xrange from io import BytesIO @@ -37,7 +36,7 @@ class TestDBBase(TestBase): """Base class providing testing routines on databases""" # data - two_lines = "1234\nhello world" + two_lines = b'1234\nhello world' all_data = (two_lines, ) def _assert_object_writing_simple(self, db): @@ -83,7 +82,7 @@ def _assert_object_writing(self, db): prev_ostream = db.set_ostream(ostream) assert type(prev_ostream) in ostreams or prev_ostream in ostreams - istream = IStream(str_blob_type, len(data), BytesIO(data.encode("ascii"))) + istream = IStream(str_blob_type, len(data), BytesIO(data)) # store returns same istream instance, with new sha set my_istream = db.store(istream) @@ -99,7 +98,7 @@ def _assert_object_writing(self, db): assert info.size == len(data) ostream = db.stream(sha) - assert ostream.read() == force_bytes(data) + assert ostream.read() == data assert ostream.type == str_blob_type assert ostream.size == len(data) else: diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index a1387ee26..db930827b 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -2,7 +2,11 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php -from gitdb.test.db.lib import * +from gitdb.test.db.lib import ( + TestDBBase, + with_rw_directory, + fixture_path +) from gitdb.db import ReferenceDB from gitdb.util import ( @@ -24,8 +28,6 @@ def make_alt_file(self, alt_path, alt_list): @with_rw_directory def test_writing(self, path): - NULL_BIN_SHA = '\0'.encode("ascii") * 20 - alt_path = os.path.join(path, 'alternates') rdb = ReferenceDB(alt_path) assert len(rdb.databases()) == 0 diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 617b51c83..2d03ad30c 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -7,7 +7,7 @@ string_types = (basestring, ) text_type = unicode -def force_bytes(data, encoding="utf-8"): +def force_bytes(data, encoding="ascii"): if isinstance(data, bytes): return data @@ -23,10 +23,6 @@ def force_text(data, encoding="utf-8"): if isinstance(data, string_types): return data.decode(encoding) - if not isinstance(data, bytes): - assert False, "Shouldn't be here" - data = force_bytes(data, encoding) - if compat.PY3: return text_type(data, encoding) else: From 7fd369c8549a975efd74c312aa91194b4569b99e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 13 Nov 2014 19:49:40 +0100 Subject: [PATCH 0265/1392] setup.py works now, and binary python module can now be loaded as well. --- gitdb/_delta_apply.c | 2 -- gitdb/fun.py | 1 - setup.py | 4 ++-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/gitdb/_delta_apply.c b/gitdb/_delta_apply.c index f03e7ea6d..8b0f8e064 100644 --- a/gitdb/_delta_apply.c +++ b/gitdb/_delta_apply.c @@ -506,7 +506,6 @@ DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) // Return the amount of chunks a slice at the given spot would have, as well as // its size in bytes it would have if the possibly partial chunks would be encoded // and added to the spot marked by sdc -inline uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) { uint num_bytes = 0; @@ -559,7 +558,6 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) // destination memory. The individual chunks written will be a byte copy of the source // data chunk stream // Return: number of chunks in the slice -inline uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint size) { assert(DIV_lbound(src) <= tofs); diff --git a/gitdb/fun.py b/gitdb/fun.py index 064680adb..b7662b495 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -758,7 +758,6 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: - # NOQA from _perf import connect_deltas except ImportError: pass diff --git a/setup.py b/setup.py index 63ec5ddb3..e01c6b475 100755 --- a/setup.py +++ b/setup.py @@ -83,12 +83,12 @@ def get_data_files(self): author = __author__, author_email = __contact__, url = __homepage__, - packages = ('gitdb', 'gitdb.db'), + packages = ('gitdb', 'gitdb.db', 'gitdb.utils'), package_dir = {'gitdb':'gitdb'}, ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('smmap (>=0.8.3)'), + requires=('smmap (>=0.8.3)', ), install_requires=('smmap >= 0.8.0'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers From f26c869025bc31d3920726a63b15fdb420b1d215 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 09:43:19 +0100 Subject: [PATCH 0266/1392] Performance tests are now part of the test-suite. By default, a small repository will be tested, which doesn't take that long actually (~20s) Additionally, that way we enforce correctness tests, which didn't run by default previously. As we are handling data here, we must be sure that it's handled correctly, thus the tests should run. --- gitdb/test/lib.py | 4 +-- gitdb/test/performance/lib.py | 27 ++++++++----------- gitdb/test/performance/test_pack.py | 21 +++++++-------- gitdb/test/performance/test_pack_streaming.py | 17 ++++++------ gitdb/test/performance/test_stream.py | 20 +++++++------- 5 files changed, 43 insertions(+), 46 deletions(-) diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index d88ec8b43..ba653c91f 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -10,7 +10,7 @@ import random from array import array -from io import StringIO +from io import BytesIO import glob import unittest @@ -120,7 +120,7 @@ def make_memory_file(size_in_bytes, randomize=False): """:return: tuple(size_of_stream, stream) :param randomize: try to produce a very random stream""" d = make_bytes(size_in_bytes, randomize) - return len(d), StringIO(d) + return len(d), BytesIO(d) #} END routines diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index 5b5c40e0d..ec45cf3a7 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -4,6 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains library functions""" import os +import logging from gitdb.test.lib import TestBase @@ -12,17 +13,6 @@ #} END invariants -#{ Utilities -def resolve_or_fail(env_var): - """:return: resolved environment variable or raise EnvironmentError""" - try: - return os.environ[env_var] - except KeyError: - raise EnvironmentError("Please set the %r envrionment variable and retry" % env_var) - # END exception handling - -#} END utilities - #{ Base Classes @@ -39,14 +29,19 @@ class TestBigRepoR(TestBase): head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' #} END invariants - @classmethod - def setUpAll(cls): + def setUp(self): try: - super(TestBigRepoR, cls).setUpAll() + super(TestBigRepoR, self).setUp() except AttributeError: pass - cls.gitrepopath = resolve_or_fail(k_env_git_repo) - assert cls.gitrepopath.endswith('.git') + + self.gitrepopath = os.environ.get(k_env_git_repo) + if not self.gitrepopath: + logging.info("You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository") + ospd = os.path.dirname + self.gitrepopath = os.path.join(ospd(ospd(ospd(ospd(__file__)))), '.git') + # end assure gitrepo is set + assert self.gitrepopath.endswith('.git') #} END base classes diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index b18e31ae6..b52e46fec 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -3,9 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance tests for object store""" -from lib import ( +from __future__ import print_function + +from gitdb.test.performance.lib import ( TestBigRepoR - ) +) from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB @@ -15,8 +17,6 @@ import os from time import time -from nose import SkipTest - class TestPackedDBPerformance(TestBigRepoR): def test_pack_random_access(self): @@ -27,7 +27,7 @@ def test_pack_random_access(self): sha_list = list(pdb.sha_iter()) elapsed = time() - st ns = len(sha_list) - print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed) + print("PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed), file=sys.stderr) # sha lookup: best-case and worst case access pdb_pack_info = pdb._pack_info @@ -41,7 +41,7 @@ def test_pack_random_access(self): # discard cache del(pdb._entities) pdb.entities() - print >> sys.stderr, "PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed) + print("PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed), file=sys.stderr) # END for each random mode # query info and streams only @@ -51,7 +51,7 @@ def test_pack_random_access(self): for sha in sha_list[:max_items]: pdb_fun(sha) elapsed = time() - st - print >> sys.stderr, "PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed) + print("PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed), file=sys.stderr) # END for each function # retrieve stream and read all @@ -65,13 +65,12 @@ def test_pack_random_access(self): total_size += stream.size elapsed = time() - st total_kib = total_size / 1000 - print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed) + print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed), file=sys.stderr) def test_correctness(self): - raise SkipTest("Takes too long, enable it if you change the algorithm and want to be sure you decode packs correctly") pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time - print >> sys.stderr, "Endurance run: verify streaming of objects (crc and sha)" + print("Endurance run: verify streaming of objects (crc and sha)", file=sys.stderr) for crc in range(2): count = 0 st = time() @@ -88,6 +87,6 @@ def test_correctness(self): # END for each index # END for each entity elapsed = time() - st - print >> sys.stderr, "PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed) + print("PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed), file=sys.stderr) # END for each verify mode diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 297426303..b1001f85b 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -3,9 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Specific test for pack streams only""" -from lib import ( +from __future__ import print_function + +from gitdb.test.performance.lib import ( TestBigRepoR - ) +) from gitdb.db.pack import PackedDB from gitdb.stream import NullStream @@ -14,7 +16,6 @@ import os import sys from time import time -from nose import SkipTest class CountedNullStream(NullStream): __slots__ = '_bw' @@ -36,7 +37,7 @@ def test_pack_writing(self): ostream = CountedNullStream() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - ni = 5000 + ni = 1000 count = 0 st = time() for sha in pdb.sha_iter(): @@ -46,17 +47,17 @@ def test_pack_writing(self): break #END gather objects for pack-writing elapsed = time() - st - print >> sys.stderr, "PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed) + print("PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed), file=sys.stderr) st = time() PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) elapsed = time() - st total_kb = ostream.bytes_written() / 1000 - print >> sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed) + print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed), sys.stderr) def test_stream_reading(self): - raise SkipTest() + # raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # streaming only, meant for --with-profile runs @@ -74,5 +75,5 @@ def test_stream_reading(self): count += 1 elapsed = time() - st total_kib = total_size / 1000 - print >> sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed) + print(sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed), sys.stderr) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 929c7e537..9d695a047 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -3,9 +3,11 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance data streaming performance""" -from lib import TestBigRepoR +from __future__ import print_function + +from gitdb.test.performance.lib import TestBigRepoR from gitdb.db import LooseObjectDB -from gitdb.stream import IStream +from gitdb import IStream from gitdb.util import bin_to_hex from gitdb.fun import chunk_size @@ -15,7 +17,7 @@ import sys -from lib import ( +from gitdb.test.lib import ( make_memory_file, with_rw_directory ) @@ -49,11 +51,11 @@ def test_large_data_streaming(self, path): # serial mode for randomize in range(2): desc = (randomize and 'random ') or '' - print >> sys.stderr, "Creating %s data ..." % desc + print("Creating %s data ..." % desc, file=sys.stderr) st = time() size, stream = make_memory_file(self.large_data_size_bytes, randomize) elapsed = time() - st - print >> sys.stderr, "Done (in %f s)" % elapsed + print("Done (in %f s)" % elapsed, file=sys.stderr) string_ios.append(stream) # writing - due to the compression it will seem faster than it is @@ -66,7 +68,7 @@ def test_large_data_streaming(self, path): size_kib = size / 1000 - print >> sys.stderr, "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add) + print("Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add), file=sys.stderr) # reading all at once st = time() @@ -76,7 +78,7 @@ def test_large_data_streaming(self, path): stream.seek(0) assert shadata == stream.getvalue() - print >> sys.stderr, "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall) + print("Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall), file=sys.stderr) # reading in chunks of 1 MiB @@ -93,10 +95,10 @@ def test_large_data_streaming(self, path): elapsed_readchunks = time() - st stream.seek(0) - assert ''.join(chunks) == stream.getvalue() + assert b''.join(chunks) == stream.getvalue() cs_kib = cs / 1000 - print >> sys.stderr, "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks) + print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks), file=sys.stderr) # del db file so we keep something to do os.remove(db_file) From 232cf20efaef4218a71c348b0f59c5e59c59cbdb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 10:43:28 +0100 Subject: [PATCH 0267/1392] Fixed incorrect computation of compressed bytes read in zlib decompression stream. --- gitdb/stream.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index e32fcf380..0332df6e4 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -275,7 +275,10 @@ def read(self, size=-1): # We feed possibly overlapping chunks, which is why the unconsumed tail # has to be taken into consideration, as well as the unused data # if we hit the end of the stream - self._cbr += len(indata) - len(self._zip.unconsumed_tail) + # NOTE: For some reason, the code worked for a long time with substracting unconsumed_tail + # Now, however, it really asks for unused_data, and I wonder whether unconsumed_tail still has to + # be substracted. On the plus side, the tests work, so it seems to be ok for py 2.7 and 3.4 + self._cbr += len(indata) - len(self._zip.unconsumed_tail) - len(self._zip.unused_data) self._br += len(dcompdat) if dat: From 6f71b8a90250ed03f2e7de2e61d22e84c0fbb2ff Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 11:04:23 +0100 Subject: [PATCH 0268/1392] Fixed .travis file to allow tests to work correctly. Previously, submodules were not initalized, which could have had an effect ... . Even though it shouldn't, but lets just try it. --- .travis.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cf1d13666..db7c2bc66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,4 +6,11 @@ python: - "3.4" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) -script: nosetests +install: + - git submodule update --init --recursive + - pip install coveralls +script: + - nosetests +after_success: + - coveralls + From b9d189d35073cc80ddbfa61269c65785264880f3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 11:29:18 +0100 Subject: [PATCH 0269/1392] Added requirements.txt for pip, and optimized test-suite performance on travis. With a bit of luck, this one will just work now. --- .travis.yml | 1 - README.rst | 33 +++++++++++-------- doc/source/intro.rst | 8 ++--- gitdb/test/db/test_git.py | 4 ++- gitdb/test/lib.py | 16 +++++++++ gitdb/test/performance/__init__.py | 1 + gitdb/test/performance/test_pack.py | 5 ++- gitdb/test/performance/test_pack_streaming.py | 3 ++ gitdb/test/performance/test_stream.py | 6 ++-- requirements.txt | 2 ++ setup.py | 2 +- 11 files changed, 56 insertions(+), 25 deletions(-) create mode 100644 gitdb/test/performance/__init__.py create mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml index db7c2bc66..10b5e161b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ python: # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) install: - - git submodule update --init --recursive - pip install coveralls script: - nosetests diff --git a/README.rst b/README.rst index f97d5c31e..194e24658 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,7 @@ GitDB ===== -GitDB allows you to access bare git repositories for reading and writing. It -aims at allowing full access to loose objects as well as packs with performance -and scalability in mind. It operates exclusively on streams, allowing to operate -on large objects with a small memory footprint. +GitDB allows you to access bare git repositories for reading and writing. It aims at allowing full access to loose objects as well as packs with performance and scalability in mind. It operates exclusively on streams, allowing to handle large objects with a small memory footprint. Installation ============ @@ -23,13 +20,13 @@ From `PyPI `_ REQUIREMENTS ============ -* Python Nose - for running the tests +* Python Nose - for running the tests SOURCE ====== The source is available in a git repository at gitorious and github: -git://github.com/gitpython-developers/gitdb.git +https://github.com/gitpython-developers/gitdb Once the clone is complete, please be sure to initialize the submodules using @@ -40,17 +37,25 @@ Run the tests with nosetests -MAILING LIST -============ -http://groups.google.com/group/git-python - -ISSUE TRACKER -============= +DEVELOPMENT +=========== .. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master :target: https://travis-ci.org/gitpython-developers/gitdb - -https://github.com/gitpython-developers/gitdb/issues + +.. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png + :target: https://coveralls.io/r/gitpython-developers/gitdb + +The library is considered mature, and not under active development. It's primary (known) use is in git-python. + +INFRASTRUCTURE +============== + +* Mailing List + * http://groups.google.com/group/git-python + +* Issue Tracker + * https://github.com/gitpython-developers/gitdb/issues LICENSE ======= diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 8fc0ec098..434138616 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -11,9 +11,9 @@ Interfaces are used to describe the API, making it easy to provide alternate imp ================ Installing GitDB ================ -Its easiest to install gitdb using the *easy_install* program, which is part of the `setuptools`_:: +Its easiest to install gitdb using the *pip* program:: - $ easy_install gitdb + $ pip install gitdb As the command will install gitdb in your respective python distribution, you will most likely need root permissions to authorize the required changes. @@ -31,10 +31,8 @@ Source Repository ================= The latest source can be cloned using git from github: - * git://github.com/gitpython-developers/gitdb.git + * https://github.com/gitpython-developers/gitdb License Information =================== *GitDB* is licensed under the New BSD License. - -.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 56899e579..e141c2ba0 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -24,9 +24,11 @@ def test_reading(self): gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") assert isinstance(gdb.info(gitdb_sha), OInfo) assert isinstance(gdb.stream(gitdb_sha), OStream) - assert gdb.size() > 200 + ni = 50 + assert gdb.size() >= ni sha_list = list(gdb.sha_iter()) assert len(sha_list) == gdb.size() + sha_list = sha_list[:ni] # speed up tests ... # This is actually a test for compound functionality, but it doesn't diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index ba653c91f..d09b1cb8e 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -18,6 +18,7 @@ import shutil import os import gc +from functools import wraps #{ Bases @@ -30,6 +31,21 @@ class TestBase(unittest.TestCase): #{ Decorators +def skip_on_travis_ci(func): + """All tests decorated with this one will raise SkipTest when run on travis ci. + Use it to workaround difficult to solve issues + NOTE: copied from bcore (https://github.com/Byron/bcore)""" + @wraps(func) + def wrapper(self, *args, **kwargs): + if 'TRAVIS' in os.environ: + import nose + raise nose.SkipTest("Cannot run on travis-ci") + # end check for travis ci + return func(self, *args, **kwargs) + # end wrapper + return wrapper + + def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the test suceeds, but leave it otherwise to aid additional debugging""" diff --git a/gitdb/test/performance/__init__.py b/gitdb/test/performance/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/gitdb/test/performance/__init__.py @@ -0,0 +1 @@ + diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index b52e46fec..db3b48de5 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -12,13 +12,15 @@ from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB from gitdb.utils.compat import xrange +from gitdb.test.lib import skip_on_travis_ci import sys import os from time import time class TestPackedDBPerformance(TestBigRepoR): - + + @skip_on_travis_ci def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) @@ -67,6 +69,7 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed), file=sys.stderr) + @skip_on_travis_ci def test_correctness(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index b1001f85b..fe160ea54 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -12,6 +12,7 @@ from gitdb.db.pack import PackedDB from gitdb.stream import NullStream from gitdb.pack import PackEntity +from gitdb.test.lib import skip_on_travis_ci import os import sys @@ -31,6 +32,7 @@ def write(self, d): class TestPackStreamingPerformance(TestBigRepoR): + @skip_on_travis_ci def test_pack_writing(self): # see how fast we can write a pack from object streams. # This will not be fast, as we take time for decompressing the streams as well @@ -56,6 +58,7 @@ def test_pack_writing(self): print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed), sys.stderr) + @skip_on_travis_ci def test_stream_reading(self): # raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 9d695a047..84c9dea3f 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -19,7 +19,8 @@ from gitdb.test.lib import ( make_memory_file, - with_rw_directory + with_rw_directory, + skip_on_travis_ci ) @@ -42,7 +43,8 @@ class TestObjDBPerformance(TestBigRepoR): large_data_size_bytes = 1000*1000*50 # some MiB should do it moderate_data_size_bytes = 1000*1000*1 # just 1 MiB - + + @skip_on_travis_ci @with_rw_directory def test_large_data_streaming(self, path): ldb = LooseObjectDB(path) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..8a4cd3979 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +gitdb +smmap>=0.8.3 \ No newline at end of file diff --git a/setup.py b/setup.py index e01c6b475..dc142c518 100755 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ def get_data_files(self): license = "BSD License", zip_safe=False, requires=('smmap (>=0.8.3)', ), - install_requires=('smmap >= 0.8.0'), + install_requires=('smmap >= 0.8.3'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ From 0bb576427f899631fbbbb822c6d3058174f96847 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 11:37:12 +0100 Subject: [PATCH 0270/1392] Allow our clone to be deeper to help tests to work --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 10b5e161b..8a9e0afa6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,9 @@ python: - "3.4" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) +git: + # a higher depth is needed for one of the tests - lets fet + depth: 1000 install: - pip install coveralls script: From e7fdd949d0cb2c42c9217e3c7009eb28c6b53446 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 12:07:13 +0100 Subject: [PATCH 0271/1392] Now I am skipping a problematic test on travis CI. Maybe I can find a py 2.6 interpreter somewhere to reproduce it. --- .travis.yml | 2 +- gitdb/test/test_stream.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a9e0afa6..761edc19b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ git: install: - pip install coveralls script: - - nosetests + - nosetests -v after_success: - coveralls diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 671a146da..f8d9f5dcd 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -27,7 +27,8 @@ import tempfile import os - +import sys +from nose import SkipTest class TestStream(TestBase): """Test stream classes""" @@ -70,10 +71,16 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): # END handle special type def test_decompress_reader(self): + cache = dict() for close_on_deletion in range(2): for with_size in range(2): for ds in self.data_sizes: - cdata = make_bytes(ds, randomize=False) + if ds in cache: + cdata = cache[ds] + else: + cdata = make_bytes(ds, randomize=False) + cache[ds] = cdata + # end handle caching (maybe helps on py2.6 ?) # zdata = zipped actual data # cdata = original content data @@ -121,6 +128,9 @@ def test_sha_writer(self): assert writer.sha() != prev_sha def test_compressed_writer(self): + if sys.version_info[:2] < (2,7) and os.environ.get('TRAVIS'): + raise SkipTest("For some reason, this test STALLS on travis ci on py2.6, but works on my centos py2.6 interpreter") + # end special case ... for ds in self.data_sizes: fd, path = tempfile.mkstemp() ostream = FDCompressedSha1Writer(fd) From 0dcec5a27b341ce58e5ab169f91aa25b2cafec0c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 12:39:09 +0100 Subject: [PATCH 0272/1392] It seems zlib works differently in py26, and thus requires special handling. This also explains why the tests suddenly stopped working - after all, the interpreter changed ... . --- gitdb/stream.py | 15 ++++++++++----- gitdb/test/test_stream.py | 13 +------------ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index 0332df6e4..edd6dd2b1 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -7,6 +7,7 @@ import mmap import os +import sys import zlib from gitdb.fun import ( @@ -30,6 +31,7 @@ from gitdb.utils.encoding import force_bytes has_perf_mod = False +PY26 = sys.version_info[:2] < (2, 7) try: from _perf import apply_delta as c_apply_delta has_perf_mod = True @@ -275,10 +277,14 @@ def read(self, size=-1): # We feed possibly overlapping chunks, which is why the unconsumed tail # has to be taken into consideration, as well as the unused data # if we hit the end of the stream - # NOTE: For some reason, the code worked for a long time with substracting unconsumed_tail - # Now, however, it really asks for unused_data, and I wonder whether unconsumed_tail still has to - # be substracted. On the plus side, the tests work, so it seems to be ok for py 2.7 and 3.4 - self._cbr += len(indata) - len(self._zip.unconsumed_tail) - len(self._zip.unused_data) + # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. + # They are thorough, and I assume it is truly working. + if PY26: + unused_datalen = len(self._zip.unconsumed_tail) + else: + unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) + # end handle very special case ... + self._cbr += len(indata) - unused_datalen self._br += len(dcompdat) if dat: @@ -505,7 +511,6 @@ def new(cls, stream_list): if stream_list[-1].type_id in delta_types: raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) # END check stream - return cls(stream_list) #} END interface diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index f8d9f5dcd..50db44b1d 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -27,8 +27,6 @@ import tempfile import os -import sys -from nose import SkipTest class TestStream(TestBase): """Test stream classes""" @@ -71,16 +69,10 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): # END handle special type def test_decompress_reader(self): - cache = dict() for close_on_deletion in range(2): for with_size in range(2): for ds in self.data_sizes: - if ds in cache: - cdata = cache[ds] - else: - cdata = make_bytes(ds, randomize=False) - cache[ds] = cdata - # end handle caching (maybe helps on py2.6 ?) + cdata = make_bytes(ds, randomize=False) # zdata = zipped actual data # cdata = original content data @@ -128,9 +120,6 @@ def test_sha_writer(self): assert writer.sha() != prev_sha def test_compressed_writer(self): - if sys.version_info[:2] < (2,7) and os.environ.get('TRAVIS'): - raise SkipTest("For some reason, this test STALLS on travis ci on py2.6, but works on my centos py2.6 interpreter") - # end special case ... for ds in self.data_sizes: fd, path = tempfile.mkstemp() ostream = FDCompressedSha1Writer(fd) From 59589e0fddfc9f3ea318dd7b3c0ef5b4e1bb665c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 14 Nov 2014 14:57:01 +0100 Subject: [PATCH 0273/1392] Added pypi badges [ skip ci ] --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 327d66372..8c9ae4251 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,9 @@ The package was tested on all of the previously mentioned configurations. ## Installing smmap +[![Latest Version](https://pypip.in/version/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) +[![Supported Python versions](https://pypip.in/py_versions/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) + Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: ```bash From f4b6b2508cf164be237b2fdeaca01be7153efe8c Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Sun, 16 Nov 2014 22:24:34 +0100 Subject: [PATCH 0274/1392] Add tox env for flake8 linter Most people know about pep8 which enforce coding style. pyflakes goes a step beyond by analyzing the code. flake8 is basically a wrapper around both pep8 and pyflakes and comes with some additional checks. I find it very useful since you only need to require one package to have a lot of code issues reported to you. This patch provides a 'flake8' tox environement to easily install and run the utility on the code base. One simply has to: tox -eflake8 The env has been added to the default list of environement to have flake8 run by default. Configuration tweaking is done in setup.cfg [flake8] section. The repository in its current state does not pass checks We can later easily ensure there is no regression by adjusting Travis configuration to run this env. More informations about flake8: https://pypi.python.org/pypi/flake8 --- setup.cfg | 3 +++ tox.ini | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 2a9acf13d..83a51c4b9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [bdist_wheel] universal = 1 + +[flake8] +exclude = .tox,.venv,build,dist,doc diff --git a/tox.ini b/tox.ini index e0e196418..35b813753 100644 --- a/tox.ini +++ b/tox.ini @@ -4,10 +4,14 @@ # and then run "tox" from this directory. [tox] -envlist = py26, py27, py33, py34 +envlist = flake8, py26, py27, py33, py34 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} deps = nose nosexcover + +[testenv:flake8] +commands = flake8 {posargs} +deps = flake8 From 01dac15d2d2ad916083559747c3df497921cdec9 Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Sun, 16 Nov 2014 22:26:49 +0100 Subject: [PATCH 0275/1392] pep8 linting E201 whitespace after '(' E203 whitespace before ',' E221 multiple spaces before operator E225 missing whitespace around operator E227 missing whitespace around bitwise or shift operator E231 missing whitespace after ',' E251 unexpected spaces around keyword / parameter equals W291 trailing whitespace W293 blank line contains whitespace E302 expected 2 blank lines, found 1 E303 too many blank lines (3) W391 blank line at end of file --- smmap/buf.py | 47 ++++--- smmap/exc.py | 6 +- smmap/mman.py | 249 ++++++++++++++++++------------------ smmap/test/lib.py | 22 ++-- smmap/test/test_buf.py | 7 +- smmap/test/test_mman.py | 3 +- smmap/test/test_tutorial.py | 40 +++--- smmap/test/test_util.py | 51 ++++---- smmap/util.py | 93 +++++++------- 9 files changed, 258 insertions(+), 260 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index ef9d49e46..66029cb99 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -10,12 +10,12 @@ class SlidingWindowMapBuffer(object): - """A buffer like object which allows direct byte-wise object and slicing into + """A buffer like object which allows direct byte-wise object and slicing into memory of a mapped file. The mapping is controlled by the provided cursor. - - The buffer is relative, that is if you map an offset, index 0 will map to the + + The buffer is relative, that is if you map an offset, index 0 will map to the first byte at the offset you used during initialization or begin_access - + **Note:** Although this type effectively hides the fact that there are mapped windows underneath, it can unfortunately not be used in any non-pure python method which needs a buffer or string""" @@ -23,12 +23,11 @@ class SlidingWindowMapBuffer(object): '_c', # our cursor '_size', # our supposed size ) - - - def __init__(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0): + + def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0): """Initalize the instance to operate on the given cursor. :param cursor: if not None, the associated cursor to the file you want to access - If None, you have call begin_access before using the buffer and provide a cursor + If None, you have call begin_access before using the buffer and provide a cursor :param offset: absolute offset in bytes :param size: the total size of the mapping. Defaults to the maximum possible size From that point on, the __len__ of the buffer will be the given size or the file size. @@ -44,10 +43,10 @@ def __init__(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0): def __del__(self): self.end_access() - + def __len__(self): return self._size - + def __getitem__(self, i): if isinstance(i, slice): return self.__getslice__(i.start or 0, i.stop or self._size) @@ -59,10 +58,10 @@ def __getitem__(self, i): c.use_region(i, 1) # END handle region usage return c.buffer()[i-c.ofs_begin()] - + def __getslice__(self, i, j): c = self._c - # fast path, slice fully included - safes a concatenate operation and + # fast path, slice fully included - safes a concatenate operation and # should be the default assert c.is_valid() if i < 0: @@ -91,18 +90,18 @@ def __getslice__(self, i, j): return bytes().join(md) # END fast or slow path #{ Interface - - def begin_access(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0): + + def begin_access(self, cursor=None, offset=0, size=sys.maxsize, flags=0): """Call this before the first use of this instance. The method was already called by the constructor in case sufficient information was provided. - + For more information no the parameters, see the __init__ method - :param path: if cursor is None the existing one will be used. + :param path: if cursor is None the existing one will be used. :return: True if the buffer can be used""" if cursor: self._c = cursor #END update our cursor - + # reuse existing cursors if possible if self._c is not None and self._c.is_associated(): res = self._c.use_region(offset, size, flags).is_valid() @@ -114,27 +113,25 @@ def begin_access(self, cursor = None, offset = 0, size = sys.maxsize, flags = 0) if size > self._c.file_size(): size = self._c.file_size() - offset #END handle size - self._size = size + self._size = size #END set size return res # END use our cursor return False - + def end_access(self): - """Call this method once you are done using the instance. It is automatically + """Call this method once you are done using the instance. It is automatically called on destruction, and should be called just in time to allow system resources to be freed. - + Once you called end_access, you must call begin access before reusing this instance!""" self._size = 0 if self._c is not None: self._c.unuse_region() #END unuse region - + def cursor(self): """:return: the currently set cursor which provides access to the data""" return self._c - - #}END interface - + #}END interface diff --git a/smmap/exc.py b/smmap/exc.py index f0ed7dcd8..5e90cf722 100644 --- a/smmap/exc.py +++ b/smmap/exc.py @@ -1,7 +1,9 @@ """Module with system exceptions""" + class MemoryManagerError(Exception): """Base class for all exceptions thrown by the memory manager""" - + + class RegionCollectionError(MemoryManagerError): - """Thrown if a memory region could not be collected, or if no region for collection was found""" + """Thrown if a memory region could not be collected, or if no region for collection was found""" diff --git a/smmap/mman.py b/smmap/mman.py index da6fd8153..6663687d0 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -20,36 +20,36 @@ class WindowCursor(object): """ - Pointer into the mapped region of the memory manager, keeping the map + Pointer into the mapped region of the memory manager, keeping the map alive until it is destroyed and no other client uses it. Cursors should not be created manually, but are instead returned by the SlidingWindowMapManager - - **Note:**: The current implementation is suited for static and sliding window managers, but it also means - that it must be suited for the somewhat quite different sliding manager. It could be improved, but + + **Note:**: The current implementation is suited for static and sliding window managers, but it also means + that it must be suited for the somewhat quite different sliding manager. It could be improved, but I see no real need to do so.""" - __slots__ = ( + __slots__ = ( '_manager', # the manger keeping all file regions '_rlist', # a regions list with regions for our file '_region', # our current region or None '_ofs', # relative offset from the actually mapped area to our start area '_size' # maximum size we should provide ) - - def __init__(self, manager = None, regions = None): + + def __init__(self, manager=None, regions=None): self._manager = manager self._rlist = regions self._region = None self._ofs = 0 self._size = 0 - + def __del__(self): self._destroy() - + def _destroy(self): """Destruction code to decrement counters""" self.unuse_region() - + if self._rlist is not None: # Actual client count, which doesn't include the reference kept by the manager, nor ours # as we are about to be deleted @@ -67,7 +67,7 @@ def _destroy(self): pass #END exception handling #END handle regions - + def _copy_from(self, rhs): """Copy all data from rhs into this instance, handles usage count""" self._manager = rhs._manager @@ -75,41 +75,41 @@ def _copy_from(self, rhs): self._region = rhs._region self._ofs = rhs._ofs self._size = rhs._size - + if self._region is not None: self._region.increment_usage_count() # END handle regions - + def __copy__(self): """copy module interface""" cpy = type(self)() cpy._copy_from(self) return cpy - + #{ Interface def assign(self, rhs): """Assign rhs to this instance. This is required in order to get a real copy. Alternativly, you can copy an existing instance using the copy module""" self._destroy() self._copy_from(rhs) - - def use_region(self, offset = 0, size = 0, flags = 0): + + def use_region(self, offset=0, size=0, flags=0): """Assure we point to a window which allows access to the given offset into the file - + :param offset: absolute offset in bytes into the file :param size: amount of bytes to map. If 0, all available bytes will be mapped :param flags: additional flags to be given to os.open in case a file handle is initially opened for mapping. Has no effect if a region can actually be reused. :return: this instance - it should be queried for whether it points to a valid memory region. This is not the case if the mapping failed because we reached the end of the file - + **Note:**: The size actually mapped may be smaller than the given size. If that is the case, either the file has reached its end, or the map was created between two existing regions""" need_region = True man = self._manager fsize = self._rlist.file_size() size = min(size or fsize, man.window_size() or fsize) # clamp size to window size - + if self._region is not None: if self._region.includes_ofs(offset): need_region = False @@ -117,91 +117,91 @@ def use_region(self, offset = 0, size = 0, flags = 0): self.unuse_region() # END handle existing region # END check existing region - + # offset too large ? if offset >= fsize: return self #END handle offset - + if need_region: self._region = man._obtain_region(self._rlist, offset, size, flags, False) #END need region handling - + self._region.increment_usage_count() self._ofs = offset - self._region._b self._size = min(size, self._region.ofs_end() - offset) - + return self - + def unuse_region(self): """Unuse the ucrrent region. Does nothing if we have no current region - + **Note:** the cursor unuses the region automatically upon destruction. It is recommended - to un-use the region once you are done reading from it in persistent cursors as it + to un-use the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" self._region = None - # note: should reset ofs and size, but we spare that for performance. Its not + # note: should reset ofs and size, but we spare that for performance. Its not # allowed to query information if we are not valid ! def buffer(self): """Return a buffer object which allows access to our memory region from our offset to the window size. Please note that it might be smaller than you requested when calling use_region() - + **Note:** You can only obtain a buffer if this instance is_valid() ! - - **Note:** buffers should not be cached passed the duration of your access as it will + + **Note:** buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" return buffer(self._region.buffer(), self._ofs, self._size) - + def map(self): """ :return: the underlying raw memory map. Please not that the offset and size is likely to be different to what you set as offset and size. Use it only if you are sure about the region it maps, which is the whole file in case of StaticWindowMapManager""" return self._region.map() - + def is_valid(self): """:return: True if we have a valid and usable region""" return self._region is not None - + def is_associated(self): """:return: True if we are associated with a specific file already""" return self._rlist is not None - + def ofs_begin(self): """:return: offset to the first byte pointed to by our cursor - + **Note:** only if is_valid() is True""" return self._region._b + self._ofs - + def ofs_end(self): """:return: offset to one past the last available byte""" # unroll method calls for performance ! return self._region._b + self._ofs + self._size - + def size(self): """:return: amount of bytes we point to""" return self._size - + def region_ref(self): """:return: weak ref to our mapped region. :raise AssertionError: if we have no current region. This is only useful for debugging""" if self._region is None: raise AssertionError("region not set") return ref(self._region) - + def includes_ofs(self, ofs): - """:return: True if the given absolute offset is contained in the cursors + """:return: True if the given absolute offset is contained in the cursors current region - + **Note:** cursor must be valid for this to work""" # unroll methods return (self._region._b + self._ofs) <= ofs < (self._region._b + self._ofs + self._size) - + def file_size(self): """:return: size of the underlying file""" return self._rlist.file_size() - + def path_or_fd(self): """:return: path or file descriptor of the underlying mapped file""" return self._rlist.path_or_fd() @@ -213,32 +213,32 @@ def path(self): raise ValueError("Path queried although mapping was applied to a file descriptor") # END handle type return self._rlist.path_or_fd() - + def fd(self): """:return: file descriptor used to create the underlying mapping. - + **Note:** it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), string_types()): raise ValueError("File descriptor queried although mapping was generated from path") #END handle type return self._rlist.path_or_fd() - + #} END interface - - + + class StaticWindowMapManager(object): """Provides a manager which will produce single size cursors that are allowed to always map the whole file. - + Clients must be written to specifically know that they are accessing their data through a StaticWindowMapManager, as they otherwise have to deal with their window size. - + These clients would have to use a SlidingWindowMapBuffer to hide this fact. - - This type will always use a maximum window size, and optimize certain methods to + + This type will always use a maximum window size, and optimize certain methods to accommodate this fact""" - + __slots__ = [ '_fdict', # mapping of path -> StorageHelper (of some kind '_window_size', # maximum size of a window @@ -247,26 +247,26 @@ class StaticWindowMapManager(object): '_memory_size', # currently allocated memory size '_handle_count', # amount of currently allocated file handles ] - + #{ Configuration MapRegionListCls = MapRegionList MapWindowCls = MapWindow MapRegionCls = MapRegion WindowCursorCls = WindowCursor #} END configuration - + _MB_in_bytes = 1024 * 1024 - - def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxsize): + + def __init__(self, window_size=0, max_memory_size=0, max_open_handles=sys.maxsize): """initialize the manager with the given parameters. - :param window_size: if -1, a default window size will be chosen depending on + :param window_size: if -1, a default window size will be chosen depending on the operating system's architecture. It will internally be quantified to a multiple of the page size If 0, the window may have any size, which basically results in mapping the whole file at one :param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions. If 0, a viable default will be set depending on the system's architecture. It is a soft limit that is tried to be kept, but nothing bad happens if we have to over-allocate :param max_open_handles: if not maxint, limit the amount of open file handles to the given number. - Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, + Otherwise the amount is only limited by the system itself. If a system or soft limit is hit, the manager will free as many handles as possible""" self._fdict = dict() self._window_size = window_size @@ -274,7 +274,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. self._max_handle_count = max_open_handles self._memory_size = 0 self._handle_count = 0 - + if window_size < 0: coeff = 64 if is_64_bit(): @@ -282,7 +282,7 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. #END handle arch self._window_size = coeff * self._MB_in_bytes # END handle max window size - + if max_memory_size == 0: coeff = 1024 if is_64_bit(): @@ -290,18 +290,18 @@ def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys. #END handle arch self._max_memory_size = coeff * self._MB_in_bytes #END handle max memory size - + #{ Internal Methods - + def _collect_lru_region(self, size): """Unmap the region which was least-recently used and has no client :param size: size of the region we want to map next (assuming its not already mapped partially or full if 0, we try to free any available region :return: Amount of freed regions - + **Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. If the system runs out of memory, it will tell. - + **todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): @@ -310,34 +310,34 @@ def _collect_lru_region(self, size): for regions in self._fdict.values(): for region in regions: # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-2 == 0 and + if (region.client_count()-2 == 0 and (lru_region is None or region._uc < lru_region._uc)): lru_region = region lru_list = regions # END update lru_region #END for each region #END for each regions list - + if lru_region is None: break #END handle region not found - + num_found += 1 del(lru_list[lru_list.index(lru_region)]) self._memory_size -= lru_region.size() self._handle_count -= 1 #END while there is more memory to free return num_found - + def _obtain_region(self, a, offset, size, flags, is_recursive): - """Utilty to create a new region - for more information on the parameters, + """Utilty to create a new region - for more information on the parameters, see MapCursor.use_region. :param a: A regions (a)rray :return: The newly created region""" if self._memory_size + size > self._max_memory_size: self._collect_lru_region(size) #END handle collection - + r = None if a: assert len(a) == 1 @@ -351,40 +351,40 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # like reading a file from disk, etc) we free up as much as possible # As this invalidates our insert position, we have to recurse here if is_recursive: - # we already tried this, and still have no success in obtaining + # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it raise #END handle existing recursion self._collect_lru_region(0) - return self._obtain_region(a, offset, size, flags, True) + return self._obtain_region(a, offset, size, flags, True) #END handle exceptions - + self._handle_count += 1 self._memory_size += r.size() a.append(r) # END handle array - + assert r.includes_ofs(offset) return r #}END internal methods - - #{ Interface + + #{ Interface def make_cursor(self, path_or_fd): """ - :return: a cursor pointing to the given path or file descriptor. + :return: a cursor pointing to the given path or file descriptor. It can be used to map new regions of the file into memory - + **Note:** if a file descriptor is given, it is assumed to be open and valid, but may be closed afterwards. To refer to the same file, you may reuse your existing file descriptor, but keep in mind that new windows can only be mapped as long as it stays valid. This is why the using actual file paths are preferred unless you plan to keep the file descriptor open. - - **Note:** file descriptors are problematic as they are not necessarily unique, as two + + **Note:** file descriptors are problematic as they are not necessarily unique, as two different files opened and closed in succession might have the same file descriptor id. - - **Note:** Using file descriptors directly is faster once new windows are mapped as it + + **Note:** Using file descriptors directly is faster once new windows are mapped as it prevents the file to be opened again just for the purpose of mapping it.""" regions = self._fdict.get(path_or_fd) if regions is None: @@ -392,92 +392,91 @@ def make_cursor(self, path_or_fd): self._fdict[path_or_fd] = regions # END obtain region for path return self.WindowCursorCls(self, regions) - + def collect(self): """Collect all available free-to-collect mapped regions :return: Amount of freed handles""" return self._collect_lru_region(0) - + def num_file_handles(self): """:return: amount of file handles in use. Each mapped region uses one file handle""" return self._handle_count - + def num_open_files(self): """Amount of opened files in the system""" - return reduce(lambda x,y: x+y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) - + return reduce(lambda x, y: x+y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) + def window_size(self): """:return: size of each window when allocating new regions""" return self._window_size - + def mapped_memory_size(self): """:return: amount of bytes currently mapped in total""" return self._memory_size - + def max_file_handles(self): """:return: maximium amount of handles we may have opened""" return self._max_handle_count - + def max_mapped_memory_size(self): """:return: maximum amount of memory we may allocate""" return self._max_memory_size - + #} END interface - + #{ Special Purpose Interface - + def force_map_handle_removal_win(self, base_path): """ONLY AVAILABLE ON WINDOWS On windows removing files is not allowed if anybody still has it opened. If this process is ourselves, and if the whole process uses this memory manager (as far as the parent framework is concerned) we can enforce - closing all memory maps whose path matches the given base path to + closing all memory maps whose path matches the given base path to allow the respective operation after all. The respective system must NOT access the closed memory regions anymore ! - This really may only be used if you know that the items which keep + This really may only be used if you know that the items which keep the cursors alive will not be using it anymore. They need to be recreated ! :return: Amount of closed handles - + **Note:** does nothing on non-windows platforms""" if sys.platform != 'win32': return #END early bailout - + num_closed = 0 for path, rlist in self._fdict.items(): if path.startswith(base_path): for region in rlist: region._mf.close() num_closed += 1 - #END path matches + #END path matches #END for each path return num_closed #} END special purpose interface - - - + + class SlidingWindowMapManager(StaticWindowMapManager): - """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. - Once a certain memory limit is reached globally, or if there cannot be more open file handles + Once a certain memory limit is reached globally, or if there cannot be more open file handles which result from each mmap call, the least recently used, and currently unused mapped regions are unloaded automatically. - + **Note:** currently not thread-safe ! - + **Note:** in the current implementation, we will automatically unload windows if we either cannot - create more memory maps (as the open file handles limit is hit) or if we have allocated more than + create more memory maps (as the open file handles limit is hit) or if we have allocated more than a safe amount of memory already, which would possibly cause memory allocations to fail as our address space is full.""" - + __slots__ = tuple() - - def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxsize): + + def __init__(self, window_size=-1, max_memory_size=0, max_open_handles=sys.maxsize): """Adjusts the default window size to -1""" super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) - + def _obtain_region(self, a, offset, size, flags, is_recursive): - # bisect to find an existing region. The c++ implementation cannot + # bisect to find an existing region. The c++ implementation cannot # do that as it uses a linked list for regions. r = None lo = 0 @@ -495,20 +494,20 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): hi = mid #END handle position #END while bisecting - + if r is None: window_size = self._window_size left = self.MapWindowCls(0, 0) mid = self.MapWindowCls(offset, size) right = self.MapWindowCls(a.file_size(), 0) - + # we want to honor the max memory size, and assure we have anough # memory available # Save calls ! if self._memory_size + window_size > self._max_memory_size: self._collect_lru_region(window_size) #END handle collection - + # we assume the list remains sorted by offset insert_pos = 0 len_regions = len(a) @@ -526,29 +525,29 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): #END if insert position is correct #END for each region # END obtain insert pos - - # adjust the actual offset and size values to create the largest + + # adjust the actual offset and size values to create the largest # possible mapping if insert_pos == 0: if len_regions: right = self.MapWindowCls.from_region(a[insert_pos]) - #END adjust right side + #END adjust right side else: if insert_pos != len_regions: right = self.MapWindowCls.from_region(a[insert_pos]) # END adjust right window left = self.MapWindowCls.from_region(a[insert_pos - 1]) #END adjust surrounding windows - + mid.extend_left_to(left, window_size) mid.extend_right_to(right, window_size) mid.align() - + # it can happen that we align beyond the end of the file if mid.ofs_end() > right.ofs: mid.size = right.ofs - mid.ofs #END readjust size - + # insert new region at the right offset to keep the order try: if self._handle_count >= self._max_handle_count: @@ -561,18 +560,16 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # like reading a file from disk, etc) we free up as much as possible # As this invalidates our insert position, we have to recurse here if is_recursive: - # we already tried this, and still have no success in obtaining + # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it raise #END handle existing recursion self._collect_lru_region(0) - return self._obtain_region(a, offset, size, flags, True) + return self._obtain_region(a, offset, size, flags, True) #END handle exceptions - + self._handle_count += 1 self._memory_size += r.size() a.insert(insert_pos, r) # END create new region return r - - diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 01f6cc918..67aec6333 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -13,18 +13,18 @@ class FileCreator(object): and provides this info to the user. Once it gets deleted, it will remove the temporary file as well.""" __slots__ = ("_size", "_path") - + def __init__(self, size, prefix=''): assert size, "Require size to be larger 0" - + self._path = tempfile.mktemp(prefix=prefix) self._size = size - + fp = open(self._path, "wb") fp.seek(size-1) fp.write(b'1') fp.close() - + assert os.path.getsize(self.path) == size def __del__(self): @@ -33,33 +33,33 @@ def __del__(self): except OSError: pass #END exception handling - @property def path(self): return self._path - + @property def size(self): return self._size #} END utilities + class TestBase(TestCase): """Foundation used by all tests""" - + #{ Configuration k_window_test_size = 1000 * 1000 * 8 + 5195 #} END configuration - + #{ Overrides @classmethod def setUpAll(cls): # nothing for now pass - + #END overrides - + #{ Interface - + #} END interface diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d3e51e2ee..d07b7f4eb 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -3,7 +3,7 @@ from .lib import TestBase, FileCreator from smmap.mman import ( - SlidingWindowMapManager, + SlidingWindowMapManager, StaticWindowMapManager ) from smmap.buf import SlidingWindowMapBuffer @@ -22,6 +22,7 @@ max_open_handles=15) static_man = StaticWindowMapManager() + class TestBuf(TestBase): def test_basics(self): @@ -82,7 +83,7 @@ def test_basics(self): max_num_accesses = 100 fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): - for manager, man_id in ( (man_optimal, 'optimal'), + for manager, man_id in ((man_optimal, 'optimal'), (man_worst_case, 'worst case'), (static_man, 'static optimal')): buf = SlidingWindowMapBuffer(manager.make_cursor(item)) @@ -114,7 +115,7 @@ def test_basics(self): assert manager.num_file_handles() assert manager.collect() assert manager.num_file_handles() == 0 - elapsed = max(time() - st, 0.001) # prevent zero division errors on windows + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows mb = float(1000*1000) mode_str = (access_mode and "slice") or "single byte" print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index cc5d91488..d903af681 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -15,6 +15,7 @@ import sys from copy import copy + class TestMMan(TestBase): def test_cursor(self): @@ -101,7 +102,7 @@ def test_memman_operation(self): fd = os.open(fc.path, os.O_RDONLY) max_num_handles = 15 #small_size = - for mtype, args in ( (StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), + for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): for item in (fc.path, fd): assert len(data) == fc.size diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index ccc113b4f..5c931de73 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -1,7 +1,8 @@ from .lib import TestBase + class TestTutorial(TestBase): - + def test_example(self): # Memory Managers ################## @@ -9,76 +10,75 @@ def test_example(self): # This instance should be globally available in your application # It is configured to be well suitable for 32-bit or 64 bit applications. mman = smmap.SlidingWindowMapManager() - + # the manager provides much useful information about its current state # like the amount of open file handles or the amount of mapped memory assert mman.num_file_handles() == 0 assert mman.mapped_memory_size() == 0 # and many more ... - + # Cursors ########## import smmap.test.lib fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") - + # obtain a cursor to access some file. c = mman.make_cursor(fc.path) - + # the cursor is now associated with the file, but not yet usable assert c.is_associated() assert not c.is_valid() - - # before you can use the cursor, you have to specify a window you want to + + # before you can use the cursor, you have to specify a window you want to # access. The following just says you want as much data as possible starting # from offset 0. # To be sure your region could be mapped, query for validity assert c.use_region().is_valid() # use_region returns self - + # once a region was mapped, you must query its dimension regularly # to assure you don't try to access its buffer out of its bounds assert c.size() c.buffer()[0] # first byte c.buffer()[1:10] # first 9 bytes c.buffer()[c.size()-1] # last byte - + # its recommended not to create big slices when feeding the buffer - # into consumers (e.g. struct or zlib). + # into consumers (e.g. struct or zlib). # Instead, either give the buffer directly, or use pythons buffer command. from smmap.util import buffer buffer(c.buffer(), 1, 9) # first 9 bytes without copying them - + # you can query absolute offsets, and check whether an offset is included # in the cursor's data. assert c.ofs_begin() < c.ofs_end() assert c.includes_ofs(100) - - # If you are over out of bounds with one of your region requests, the + + # If you are over out of bounds with one of your region requests, the # cursor will be come invalid. It cannot be used in that state assert not c.use_region(fc.size, 100).is_valid() # map as much as possible after skipping the first 100 bytes assert c.use_region(100).is_valid() - + # You can explicitly free cursor resources by unusing the cursor's region c.unuse_region() assert not c.is_valid() - + # Buffers ######### # Create a default buffer which can operate on the whole file buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) - + # you can use it right away assert buf.cursor().is_valid() - + buf[0] # access the first byte buf[-1] # access the last ten bytes on the file buf[-10:]# access the last ten bytes - + # If you want to keep the instance between different accesses, use the # dedicated methods buf.end_access() assert not buf.cursor().is_valid() # you cannot use the buffer anymore assert buf.begin_access(offset=10) # start using the buffer at an offset - + # it will stop using resources automatically once it goes out of scope - diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 745da83d7..745fedf2e 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -12,18 +12,19 @@ import os import sys + class TestMMan(TestBase): - + def test_window(self): wl = MapWindow(0, 1) # left wc = MapWindow(1, 1) # center wc2 = MapWindow(10, 5) # another center wr = MapWindow(8000, 50) # right - + assert wl.ofs_end() == 1 assert wc.ofs_end() == 2 assert wr.ofs_end() == 8050 - + # extension does nothing if already in place maxsize = 100 wc.extend_left_to(wl, maxsize) @@ -31,34 +32,33 @@ def test_window(self): wl.extend_right_to(wc, maxsize) wl.extend_right_to(wc, maxsize) assert wl.ofs == 0 and wl.size == 1 - + # an actual left extension pofs_end = wc2.ofs_end() wc2.extend_left_to(wc, maxsize) - assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() - - + assert wc2.ofs == wc.ofs_end() and pofs_end == wc2.ofs_end() + # respects maxsize wc.extend_right_to(wr, maxsize) assert wc.ofs == 1 and wc.size == maxsize wc.extend_right_to(wr, maxsize) assert wc.ofs == 1 and wc.size == maxsize - + # without maxsize wc.extend_right_to(wr, sys.maxsize) assert wc.ofs_end() == wr.ofs and wc.ofs == 1 - + # extend left wr.extend_left_to(wc2, maxsize) wr.extend_left_to(wc2, maxsize) assert wr.size == maxsize - + wr.extend_left_to(wc2, sys.maxsize) assert wr.ofs == wc2.ofs_end() - + wc.align() assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) - + def test_region(self): fc = FileCreator(self.k_window_test_size, "window_test") half_size = fc.size // 2 @@ -66,56 +66,55 @@ def test_region(self): rfull = MapRegion(fc.path, 0, fc.size) rhalfofs = MapRegion(fc.path, rofs, fc.size) rhalfsize = MapRegion(fc.path, 0, half_size) - + # offsets assert rfull.ofs_begin() == 0 and rfull.size() == fc.size assert rfull.ofs_end() == fc.size # if this method works, it works always - + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size - + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) - # with the values we have, this test only works on windows where an alignment + # with the values we have, this test only works on windows where an alignment # size of 4096 is assumed. - # We only test on linux as it is inconsitent between the python versions + # We only test on linux as it is inconsitent between the python versions # as they use different mapping techniques to circumvent the missing offset # argument of mmap. if sys.platform != 'win32': assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) #END handle platforms - + # auto-refcount assert rfull.client_count() == 1 rfull2 = rfull assert rfull.client_count() == 2 - + # usage assert rfull.usage_count() == 0 rfull.increment_usage_count() assert rfull.usage_count() == 1 - + # window constructor w = MapWindow.from_region(rfull) assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() - + def test_region_list(self): fc = FileCreator(100, "sample_file") - + fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): ml = MapRegionList(item) - + assert ml.client_count() == 1 - + assert len(ml) == 0 assert ml.path_or_fd() == item assert ml.file_size() == fc.size #END handle input os.close(fd) - + def test_util(self): assert isinstance(is_64_bit(), bool) # just call it assert align_to_mmap(1, False) == 0 assert align_to_mmap(1, True) == ALLOCATIONGRANULARITY - diff --git a/smmap/util.py b/smmap/util.py index 44e94124d..137c19d7b 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -12,7 +12,7 @@ from mmap import PAGESIZE as ALLOCATIONGRANULARITY #END handle pythons missing quality assurance -__all__ = [ "align_to_mmap", "is_64_bit", "buffer", +__all__ = ["align_to_mmap", "is_64_bit", "buffer", "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities @@ -27,6 +27,7 @@ def buffer(obj, offset, size): # doing it directly is much faster ! return obj[offset:offset+size] + def string_types(): if sys.version_info[0] >= 3: return str @@ -37,7 +38,7 @@ def string_types(): def align_to_mmap(num, round_up): """ Align the given integer number to the closest page offset, which usually is 4096 bytes. - + :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" @@ -46,15 +47,16 @@ def align_to_mmap(num, round_up): res += ALLOCATIONGRANULARITY #END handle size return res; - + + def is_64_bit(): """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" - return sys.maxsize > (1<<32) - 1 + return sys.maxsize > (1 << 32) - 1 #}END utilities -#{ Utility Classes +#{ Utility Classes class MapWindow(object): """Utility type which is used to snap windows towards each other, and to adjust their size""" @@ -68,7 +70,7 @@ def __init__(self, offset, size): self.size = size def __repr__(self): - return "MapWindow(%i, %i)" % (self.ofs, self.size) + return "MapWindow(%i, %i)" % (self.ofs, self.size) @classmethod def from_region(cls, region): @@ -86,7 +88,7 @@ def align(self): self.size = align_to_mmap(self.size, 1) def extend_left_to(self, window, max_size): - """Adjust the offset to start where the given window on our left ends if possible, + """Adjust the offset to start where the given window on our left ends if possible, but don't make yourself larger than max_size. The resize will assure that the new window still contains the old window area""" rofs = self.ofs - window.ofs_end() @@ -103,10 +105,10 @@ def extend_right_to(self, window, max_size): class MapRegion(object): """Defines a mapped region of memory, aligned to pagesizes - + **Note:** deallocates used region automatically on destruction""" __slots__ = [ - '_b' , # beginning of mapping + '_b', # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_uc', # total amount of usages '_size', # cached size of our memory map @@ -117,32 +119,31 @@ class MapRegion(object): if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset #END handle additional slot - + #{ Configuration # Used for testing only. If True, all data will be loaded into memory at once. # This makes sure no file handles will remain open. _test_read_into_memory = False #} END configuration - - - def __init__(self, path_or_fd, ofs, size, flags = 0): + + def __init__(self, path_or_fd, ofs, size, flags=0): """Initialize a region, allocate the memory map :param path_or_fd: path to the file to map, or the opened file descriptor - :param ofs: **aligned** offset into the file to be mapped + :param ofs: **aligned** offset into the file to be mapped :param size: if size is larger then the file on disk, the whole file will be allocated the the size automatically adjusted - :param flags: additional flags to be given when opening the file. + :param flags: additional flags to be given when opening the file. :raise Exception: if no memory can be allocated""" self._b = ofs self._size = 0 self._uc = 0 - + if isinstance(path_or_fd, int): fd = path_or_fd else: - fd = os.open(path_or_fd, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + fd = os.open(path_or_fd, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags) #END handle fd - + try: kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size @@ -152,8 +153,8 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): corrected_size += ofs sizeofs = 0 # END handle python not supporting offset ! Arg - - # have to correct size, otherwise (instead of the c version) it will + + # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) @@ -162,9 +163,9 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): else: self._mf = mmap(fd, actual_size, **kwargs) #END handle memory mode - + self._size = len(self._mf) - + if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, self._size) #END handle buffer wrapping @@ -173,7 +174,7 @@ def __init__(self, path_or_fd, ofs, size, flags = 0): os.close(fd) #END only close it if we opened it #END close file handle - + def _read_into_memory(self, fd, offset, size): """:return: string data as read from the given file descriptor, offset and size """ os.lseek(fd, offset, os.SEEK_SET) @@ -186,92 +187,92 @@ def _read_into_memory(self, fd, offset, size): mf += d #END loop copy items return mf - + def __repr__(self): return "MapRegion<%i, %i>" % (self._b, self.size()) - + #{ Interface def buffer(self): """:return: a buffer containing the memory""" return self._mf - + def map(self): """:return: a memory map containing the memory""" return self._mf - + def ofs_begin(self): """:return: absolute byte offset to the first byte of the mapping""" return self._b - + def size(self): """:return: total size of the mapped region in bytes""" return self._size - + def ofs_end(self): """:return: Absolute offset to one byte beyond the mapping into the file""" return self._b + self._size - + def includes_ofs(self, ofs): """:return: True if the given offset can be read in our mapped region""" return self._b <= ofs < self._b + self._size - + def client_count(self): """:return: number of clients currently using this region""" from sys import getrefcount # -1: self on stack, -1 self in this method, -1 self in getrefcount return getrefcount(self)-3 - + def usage_count(self): """:return: amount of usages so far""" return self._uc - + def increment_usage_count(self): """Adjust the usage count by the given positive or negative offset""" self._uc += 1 - + # re-define all methods which need offset adjustments in compatibility mode if _need_compat_layer: def size(self): return self._size - self._b - + def ofs_end(self): # always the size - we are as large as it gets return self._size - + def buffer(self): return self._mfb - + def includes_ofs(self, ofs): return self._b <= ofs < self._size #END handle compat layer - + #} END interface - - + + class MapRegionList(list): """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( '_path_or_fd', # path or file descriptor which is mapped by all our regions '_file_size' # total size of the file we map ) - + def __new__(cls, path): return super(MapRegionList, cls).__new__(cls) - + def __init__(self, path_or_fd): self._path_or_fd = path_or_fd self._file_size = None - + def client_count(self): """:return: amount of clients which hold a reference to this instance""" from sys import getrefcount return getrefcount(self)-3 - + def path_or_fd(self): """:return: path or file descriptor we are attached to""" return self._path_or_fd - + def file_size(self): """:return: size of file we manager""" if self._file_size is None: @@ -282,5 +283,5 @@ def file_size(self): #END handle path type #END update file size return self._file_size - + #} END utility classes From 1cd3760be374521a7fafe2262e58c1703b71b9aa Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Sun, 16 Nov 2014 22:37:15 +0100 Subject: [PATCH 0276/1392] Drop semicolon at end of statement Fix pep8: E703 statement ends with a semicolon --- smmap/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/smmap/util.py b/smmap/util.py index 137c19d7b..394e6b197 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -42,11 +42,11 @@ def align_to_mmap(num, round_up): :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" - res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY; + res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY if round_up and (res != num): res += ALLOCATIONGRANULARITY #END handle size - return res; + return res def is_64_bit(): From eb40b44ce4a6e646aabf7b7091d876738336c42f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 19 Nov 2014 17:12:38 +0100 Subject: [PATCH 0277/1392] Added link to readthedocs --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8c9ae4251..e15c9bfc5 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ The package was tested on all of the previously mentioned configurations. [![Latest Version](https://pypip.in/version/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) [![Supported Python versions](https://pypip.in/py_versions/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) +[![Documentation Status](https://readthedocs.org/projects/smmap/badge/?version=latest)](https://readthedocs.org/projects/smmap/?badge=latest) Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: From ab4520683ab325046f2a9fe6ebf127dbbab60dfe Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 19 Nov 2014 17:19:52 +0100 Subject: [PATCH 0278/1392] Added readthedocs badge --- README.rst | 9 ++++++--- gitdb/ext/smmap | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 194e24658..186218d1f 100644 --- a/README.rst +++ b/README.rst @@ -12,6 +12,9 @@ Installation .. image:: https://pypip.in/py_versions/gitdb/badge.svg :target: https://pypi.python.org/pypi/gitdb/ :alt: Supported Python versions +.. image:: https://readthedocs.org/projects/gitdb/badge/?version=latest + :target: https://readthedocs.org/projects/gitdb/?badge=latest + :alt: Documentation Status From `PyPI `_ @@ -44,7 +47,7 @@ DEVELOPMENT :target: https://travis-ci.org/gitpython-developers/gitdb .. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png - :target: https://coveralls.io/r/gitpython-developers/gitdb + :target: https://coveralls.io/r/gitpython-developers/gitdb The library is considered mature, and not under active development. It's primary (known) use is in git-python. @@ -52,10 +55,10 @@ INFRASTRUCTURE ============== * Mailing List - * http://groups.google.com/group/git-python + * http://groups.google.com/group/git-python * Issue Tracker - * https://github.com/gitpython-developers/gitdb/issues + * https://github.com/gitpython-developers/gitdb/issues LICENSE ======= diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 28fd45e0a..eb40b44ce 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 28fd45e0a7018f166820a5e00fce2ccb05ebdb61 +Subproject commit eb40b44ce4a6e646aabf7b7091d876738336c42f From 5e3dbccb05f85f178d3260a27df2f59a69221668 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 17 Dec 2014 23:09:42 -0500 Subject: [PATCH 0279/1392] Bring gitdb.test back into distribution --- MANIFEST.in | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index b14aed9b3..597944fd4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,7 +8,7 @@ include gitdb/_fun.c include gitdb/_delta_apply.c include gitdb/_delta_apply.h -prune gitdb/test +graft gitdb/test global-exclude .git* global-exclude *.pyc diff --git a/setup.py b/setup.py index dc142c518..1d327eba0 100755 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def get_data_files(self): author = __author__, author_email = __contact__, url = __homepage__, - packages = ('gitdb', 'gitdb.db', 'gitdb.utils'), + packages = ('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), package_dir = {'gitdb':'gitdb'}, ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", From c38bd19706abe5cf0bf0e7b3e9ad2b3e554d28ef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jan 2015 13:47:19 +0100 Subject: [PATCH 0280/1392] Increased initial size of decompressed data to obtain loose object header information This appears to fix https://github.com/gitpython-developers/GitPython/issues/220 , in this particular case. Nonetheless, we might just have gotten lucky here, and the actual issue is not yet solved and can thus re-occour. It would certainly be best to churn through plenty of loose objects to assure this truly works now. Maybe the pack could be recompressed as loose objects to get a sufficiently large data set --- gitdb/stream.py | 7 +++++-- .../88/8401851f15db0eed60eb1bc29dec5ddcace911 | Bin 0 -> 222336 bytes gitdb/test/performance/test_pack.py | 3 ++- gitdb/test/test_stream.py | 13 ++++++++----- 4 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 gitdb/test/fixtures/objects/88/8401851f15db0eed60eb1bc29dec5ddcace911 diff --git a/gitdb/stream.py b/gitdb/stream.py index edd6dd2b1..b0a89002a 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -100,7 +100,9 @@ def _parse_header_info(self): :return: parsed type_string, size""" # read header - maxb = 512 # should really be enough, cgit uses 8192 I believe + # should really be enough, cgit uses 8192 I believe + # And for good reason !! This needs to be that high for the header to be read correctly in all cases + maxb = 8192 self._s = maxb hdr = self.read(maxb) hdrend = hdr.find(NULL_BYTE) @@ -243,7 +245,7 @@ def read(self, size=-1): # moving the window into the memory map along as we decompress, which keeps # the tail smaller than our chunk-size. This causes 'only' the chunk to be # copied once, and another copy of a part of it when it creates the unconsumed - # tail. We have to use it to hand in the appropriate amount of bytes durin g + # tail. We have to use it to hand in the appropriate amount of bytes during # the next read. tail = self._zip.unconsumed_tail if tail: @@ -284,6 +286,7 @@ def read(self, size=-1): else: unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) # end handle very special case ... + self._cbr += len(indata) - unused_datalen self._br += len(dcompdat) diff --git a/gitdb/test/fixtures/objects/88/8401851f15db0eed60eb1bc29dec5ddcace911 b/gitdb/test/fixtures/objects/88/8401851f15db0eed60eb1bc29dec5ddcace911 new file mode 100644 index 0000000000000000000000000000000000000000..d60aeeff74d2983bcc2abd03163d2cc8c422f819 GIT binary patch literal 222336 zcmV(zK<2-A0gReca4t*`reoW-ZQHhO<0ScF+qO<@+qP{xIkBDWzq;OiYieq`rn)cY zrl0C}Oza#@h*(%ySedzii2012O|1lA%*;)!jUDJstR0Me6?kAY$rk__!v%mWvr>c|!t=BR(^2i-&|PE)rDF zfk!^fRVBjq@wl-S!7GHO6AF3#{Ejj{^|X?hEje2mmunc@?hfXE+wMbLFsSn}W!6+7 zrYYW(|IjgGY6HhALjLvWd9t14+jh1M1wY}9KMQ@OOYlBu(6qS&Rz zG;LAnQ9Rj@lNGbDc1h^FSj3=9lCpHf@k2234o%CG%}5xx#lpPiD%5;uG3Xu2Yr(BE zon1Lm&JI?IPN%}Rb~b9`J+dXv>-8lU_vK#mm(pQQ7@{r{q2nMeA~qv@Gw_PqX;EKR zYJva*$1;8Hs{v+8yqVOgCAu|<0k-zn#_`2qrpm?Y)|1sb9eYEQUIOo7(7eT7IW>rD z?E~Qg7s|}D(Xf6qCW&m3kjSFF{$`C%`>Mr%1$O^_v}k3|KGQ{)D8SLPAkOhWPZ`cC zTr<&Dd2M?Xsd*_}V)+fm|3y>!`!?`BmM%X#f18>Fv0%gV(M0m5FYVjc^TBhVZhN>B zu4pO}qYsQ01JyGW&j?Y^0~ts%-Lbxx#!VtmBwzj1&3VWV+GKHGO(qNvzo#vR{&FYp3$w zc4GpwFu}I_h6%VBt7ja9Tfabd+ShP7$JuqZ4NNSYs=uu z8wE?Q0BA=a`2`jP95%&u>kb;CbZ1m4k@}HhSjcNEx3x!DVY}MHHLf!|bJZ+Em$9b~ za&;zCbTC$oV9E|m>Db+qog2;$-Bc#Zk5GW8B!-f~&+h2pA0L5N?ap_KsrSmh5LCus zCy`swgR^0vIWd*ul+cD(_e5zPdfG)xJJ1-k0`oygR|SN+^@J7@q_VR2Zz$AF>TcG1 zjg&v}aMCNNokyrbrU61s$f0t##PrL(?V`#ZZECh)u2>uZ^BN6doe+HpOO*$L#od20 z*^|GFj{XSQugs$!^!hZ#Ei#Y}Y|CfljmV~fgs6_L_=0xNY&btn3wP;sUJ61y&{5X$ z&tMgRGaF_77+>&!0GUI#G?SYC!wqn$p2-XzDA1k!dq)a>jgj!*2gA@%<7$#qnE$5HR4@*Evg7r zkckvjr^Kk@L}OWM&w^2fn*4Wj_B&+qFl`2fIX?VXcNltJq&X9*LwxMorn43jK}QBZ zSq@8!6<~+-%Q|Ni-7(!Wy?w32Q;A5C2BN*ntp*mcU0%DGLrmZ?f^HVXle;tEtj;J$ zH;rDQG*EySc(1En46dl$@+Sp^V&=StZtZ!WNYvg#|`S9#W5sVWE((c zcH3nuNGKume`Uz5gYrv)7;X!?f7h4+wIsk5xfVHM$7eY-hS`oJukZG z(SP#jJ^tYN{p%IuZuhqA!8iHIj|eueTaw^53?dVZ+*Q4Yq&!P`HA{2Y{zrQiNkd)S zUv^UHM*_q9;qDXmcocr41!2Py`O|BVuqH>)aHXsAi@k+OyK zBMsWy(Ny?X*J;T8;y2g7>E<{2hq392{kJT0(&E>@>&G7oh%8g@^WgJ}_&tU1>Z_Ie z_0J*M#~+b^*YRbk{kk`}-l~J1vtB}P(WS&z+tuRgiGnP$@XZuF8fIs8!X(j9$sd8Z z*bEt5XKkv)3KpX9FFAx02yQe+Le#Z{s)leVmu`wiEj*3SpZ3a?lh~2BTMuJo$1-0(qT)fX#lj^aI!=B?59>;u@NwcZ!}G-inz2$f5yh&#V% z!=CF&xAS!Aq%V;#LL98lQZM({K0Oszz;E{GSDv%L=kHek-bbzE13&qD zg^tCWDWJg3LKBfUk?-aT;;$`a`!)4~ZkgDFT>9N4ikrVoFZLd(<KAEZ5If zgsFSkRZptUwr?HYJd`+iPctHk^neYis^(M41lYT+*T)`{$2Tw?qD$I9E_K6NPZ<{b zW#X~cuG87WRVItW?SjU$wJOXRS^k8L0?e@X@_24$wXLYeKCFOWzlMQ~Gt7;T?m$?Pt6|Lt2uopo}W538|(i;XW-I~~7f3i1;Bd;T0hUU&~6zbK49vNj9sv57vGv3|XQ z^vu=O&=NEDe*5S=ttkqA+DZ2gG{(Qie8)~!j?#x!hIH6X)!ODi*J6*Zdgz}BhXO*W zFgqx|O?JZhjUiq^gZ712i)4pKTgB-K+BWmGZgm}H%TwQw^8KDjCwSMz*z$>W4Dd5I zOTAhgY`#RC>9z<^qE&zH1%H1ULVS!a&Wh>@NnHtOTL{e^6J(HekjUyM<=3uO*R0*5 z*YMX9d`&_BhI67!3-y{_E#kO>gA;v*XVePvp}`ZkqPEjurgV%2FkYvH9dl z!_8}ET#J4lNS(Si5jr9^yQmb?Fzmc2#Fh9w*YuIeoc(Rs^|X8zluh;c{$lLNc>HB7 z{E?&+;}EL#zlORSMC^Kzcox+7Az<*qE^>;He2(WrQ+!qjUBi*JOi0-Y1b65Oo*Hew zljY~fn+2)E$BQeCX}`2Xb{Vv-W59@5=ZdR#=m5_h9f6(SdMk_T8q?SR+nWtKGuO%8 zInzB3N%*zRTXO6^c1!lVmb&Ou#%4MigUxC&ox%M7b2!qB{sobdvPuC`0s=BK1{nV@ z)b;oeb^p`P!%Ta)Z^fB+E#WBf3Wr5bAg-+3JM>NzVyrc=S z6WKXBomuGkBiCkTGLyJJ!W&p9#o0El_Y8E^1=pV+I$hYxY@9Zkwm2;?Oci1INw|ka z`r?Dv447~fJEU(sNPF}P4F6^FA{j`NN4u$k+9q$9)g-Ir?j06JX~=5-S3?U@(~# zb|GVqm&EDHKE*2vh$6Ry>-x7r$|}PrUEPJnvtLU?P87dd1)4Ol&P>a4owNEXTt|bu z>e;y{uey|dID~k`XN*9378)iMtF+KF1Zib(B`4TYsQ$^1uVHVWVO$3NxV!X1Nr6+% z>c=YA_?LHy#l`Q5U5p)uTB=Td)yBP5z!wlHc3{SM`Lj#J#-t==qbTLWWC#G8*EZRb z#iEu->z;KUwS!dR0u?SBoRov;nv!5Fzv~@YV@J55Rb@jk4!toXz+wY;$j*PJhI~gu z#{~CCRMQ*nRU0`bjECEq-^k195jn=PM33EFo&;7CX1<4RppoMDdqo^g>ws4bxlW+( zgS+)=b_r=GH{i1-24BCcXmdZ$%i&+B4O2&+P$+{IJFf*5nAShRoh2-z(Soy1AQmTl zH&6p2wYDS6^`qy+^K+ygG8K>A9u{t)a<&lyfQ1=F(HGw_q?1mr;v6XQ7_0CHZb~=-IfVOZ0LNKvfJkZ`__*P-3aO>MzP!_GMe#$L zk0fF_RA2p#d#NS!a}-K{e>xBl0WIgrmf>pNWn;AA+ONdPaXxhNIE1&F)~enU#O;)_C?Lf^_e?O&eq6I;xuV!cL+sy8m*^d)^j0 z-N&YbCphB?inuKB-nnf7MUi%#zA4=zRE`@d8$|(_y;jv4=88PyfO=}4^?NA>M2)8j zW9B=a;hVo0Bqfw2IVY+^b?P%G;VJbfVODxAT@Dp_?9`W!NmK5IvO&&l3ElDU`upe0 zZ8y_MCU#EgspzjP@ItG8o`@ESy21}9a(Eb-QJS}| zHgCv>h2JIbjR@SpcusNqQRKo5>XlUo>;tt&Uo!MVB#&;x4(5x#Oc|`5W2_gFg~dKR zX-rFLWm@r^@M^1=or+AjEvWxi$u{rNAKp?_P2zq{meK+NZWA#SI5`)ot(yZ&tzy(y z;sH?Aa|s&g;3@@J@(Uq^2u*9oi|4u}FrST=AZ&Ut%_RWOB&c>GpKPuwcmkdHj=x-W zTDPuS4p~YVWep-Y*%YVHcQc7Mf8KS`6>fO2372*j&gf8`y$NP4DT5kp%> z(0W>4T`@YgpLi;8y@+be?XycShpWi6CjAcs!D=aa2Ic>* zQ;gT>i^zv%#Cj%cIT;2x5mN!Tl=OF@VN*Ho(D5I<+Nq- zKt$IZdN>4S%ro_EE>6iiUM~FWgv*qy08k^X(UmEGfQEBg1DY$QV|(W93N379e&5D9 zpji!szoxOQ7$2wQaq@q`q8A*C5vD=x&;dP|u*UnkSr2CQya>11($=f4)q%O^rS z5=FT@+?*4OE;39f>YM1gTw6AspaPMjFlE(amcfG~(ugNu12q-#W3B9Qw5{xCgLs+_ zfrRyrhO{3papdP&>&dIv$DV;P-Hs$-XI#CAQ7sBJz;a-zlw?5s)&}yPQU-xe0EL{w z)3@dIAb(7{vEr#H#QORIa@;wI0o;*iqmB$VJEj^&;=F)W4Pt`fNr?d34*85c59i)L zkXgw^jd&Blnas@4NT+tm1?RE+TWi8>hSGN+v2YRrTXhnm0o9A5vb7{L@Jv}nnIQij zY5epaAlh7k!#vRW4L!K|XhiG7J>x%uz8%Y+{{W9ypyINDjUm5Y*3A975X48fYTZD? z@wE!gBATVEEg>;D@A*N5(v+G4Zg~~64Me#SZ0TpdR@OYG-!p0oIQp_w^_10;o}MUt zXWd_APHhXZC4LKpN9vc=qLJH+C6H-ufB!L68wS`rD%`zw{NawHt<-%&T1s_>fUzuU zh$sQ!T|@v9kAfa$S*jY{^K?FwP-{E5h@j#+3hH$C&F(aGI_C7tsQ?#d)zIDP(3##I z6P;^BxMdZED6PgqYo12aQviwzv$&|Xs9)X>o1Ify54CGNZ*PARp9;U-Y7*!bj6Efr zO!q|c_3d?@m=;Y=^i;xqNC8ENt(iGiW>yWJ98CKp)F8Lz(K&j%Q>i4KM4rx^-bh5o zefNc&Pp(vxxen?Ois*nq(#T=6C77D`6tv}(ga4>AQ(ju*mF9{Svk|%b%~!2|<0{d{ zA-uy{jJDj=RiMhke`la9!nn&2_7Fl{^_Y3_w*iN$(B9n%VQO5($H;00Hy4@NAI+JX zWi&4vH8H1Fnl&1)rqJhDdGif0ZqXfw@bmSrCULRGD-;R-|)d)2}moJLRZ* zt=#K@&Tpe}60U-0eMyNV<+of~a}l4P4^njtRxwG`vv;Q<<1XT1APHY*(;z#wWz61Q zR(@T}d9TCyvLU|fw>dB!IEwJ9Gmrs(5WUBw1=%qt4>#v2eTV4}1ZC@;r_#3&1xG=g zSPtOAP+DtrMr`(jtb%~V^Z4yKs8TQik!AL?r;2|cS z5TfN_UDQsqCS{l^Y>KI&R8YlUf=;k&IVcm6qS%GaW-WiW|Mf2Kp1 z-2z77*hmR6Wr<%qm`r7wDwjaRi|8gv*GR)X1qo-UU2ZMK6jcp6BKh*!(vAq#ZAB|o zcrO7xW88Lm`Znm#`P1gTB6h&G*vS{syqVWjY$!8v(ARvbx>~cAspv~{W|zTx+{L&O zw&_qf?_E*{!s)LF)5hKw3?Uzv9o1yKi`H|ZfO=#7+N)ke0bLm_lU1);Rf)nDuFCnE z(ekU1^n5pFGdahQZzD#MpWDNJiA|@rW5|nnVhXsppnkgPo476Yk#&!lWwyXrbNslb z<&e2-#o`R43@emNZ67ePcfiU-)=j%f{F7Jp*~rHnm0NXnS9BZn5ue%d{`!l5;BFPE z`XgBQcpvD>DC=co&l7~M-ey&x@aHenqZ6GAW~~2l^?As})Otf+j;0ofoBqs?A21EY z#;Fy6x-&iFG7t(QtQhNEl5&K_=Xg{yBFK0NdI@y)C%pmNYF%!5;H(Fye`Fk$EVAzB zaBEvb*NBG#b8XUE>Yv4~lFO^PU!z(-AmvC|Y-6+WJ0N-HV>TQgy@gA~%^}J$2)KnU zS+lt*+Eanj3R(qe*8L2M3Dr`zEUNEW^sIx#mXG%wwLrD#kC z+Ci?O22Y- zuNYVBZ1B0=o`#|Aufn*@F&n-+x`M>+!Mo_2d3P>FL8zAKH1dENwnB7J%mlSpgIrSE zO8Et3$LbUXJitZU&G8pLp6CQ-L4{Mh>wZ2e^a)JsT~-f#0{-IhH($A5cd9ACr_jJC zY+tSoXz_)n6-iK7Bekxr%K){ZI4eX{GT33V92TPcni1Z^Nz(Io%873|Hy>~Y@(j|) zMtB_!@y7b&pMp3}n?j(w2NjAaXRcuWR>r>=90s@B+%oDO&6a1ZS$fPhDMHS|8w0(H zqVE>y{DY+NBw2+i>1|E|F$hMFMR*9}K>Xf^CLWQCFb;;ZB^wN$0 zEF6tRAcQD;8&`BEiReYc@FaO}>(hw!+TK+n;Lx;N2DHtzVloEb-S_puwV7H^y|2dI)%f;zj$ z0flpF!}1JMK6^LanpNo|MjS>Lg-|z^d@@om4MdiIcQYj-Azz%z(8gvUO9(I1l9bez z-}>w>(Ljn^_2ce5kK<)<_c~MWFVL>fZTQtW&l?GA6q04P%rkTf5{r$8#0BXOulW34 z-IW!;=&td36f4>J4e1`oKZQM@9_M(JAlzn~kmscF9UxvH6Pf@WbN7`KjHgdS2C->T zAQ2ap!Df78*dvUvzw!Vba4cx=k}AnA(?&Z!t?~~%4Exex^MUf+qV{CY7aJqfq-=({ zc$S+2ckG<(<btKP-7^jmT89<5y9l|3ZH-3HT7hi%N^Q zGJ<08m*nlcxuJR>G+}m<4-oUQC&TXexM3Wx9!M|B7=OLH$>yU)Bq&~64s9`%WzFoEiYz(c3E9#y2nkE571~0IXdCR zhW1C6a}VrupK!f;gI=6pQnlYN==bRm*CfC4gpG@LW#!ce)r+0D@1KdmJ!~OCO%I&H z4Hb+P%`y%Jmnq&b6!e9tgoVE((4S`G*;B=*dW|kt8Y?fcJID?|sPf|2ov4+{)AgX; z*wytF^F&C~NETQ#t!Uk0im0^=Na&Ey=%eAO@{rFg#2MOcg0=#prvfN$Hff*};MmO2 z^Kd3;0sY8nhTw7#IDqAUEtO3=#1q6bR#easG7Cd9x^A6^aWhNQ(6K;<)uHU*fmh^W zJHp1ZG#Q6dYGp&Xoiuby)P1UxS>Ym2Jfs`7gU&Ys_Fg=09Zugcae*9vjYLrl&owF8 zE%S6^3vT_;jB0r=rJ&euE6R6KNzvb{WLuP4lg3h&lPk>0RixQ7?XK;e=H8;1h51}> zFeo8hcr*Mw#l)@Mcn4HRr25qt{w2D5pU1-MaoY zYVE?c$?&m-I)Az3*TL>Xw^2(qe1D_f7MMm;4~22xWn#E3DriI~nvGr$;n}9r_q^2H z(MZqVFX~A~mo!Gzw|87lDddorAlAj_hSeIJ*6&tS*|NvJmVyb|E_Wx`Bg1T(J`xi2 z0qfK%-KSe_Hm$kp0lavCLXTdsfSNu$>??(^etgGg7Rf@wm||JGM31P}cGrwvu5_4` z)@GiuMsLZ>Ox*@eqFXM`Z4#C{9Abr0!!8DU>}TDgG(q!(qpEK$CveNuKbC|s%onPW z9@>5kk6|x&TnQQWEQ=a_5eZvkMzv#G?E_)Kj@6keR^sk5ti;^!HBdM=Zta2k zUO1?~M%!;%YpfrP3=kNmP`U(?T7%Y`3K#7&4EFI)l2C^F2&kW>{n=~I7wtA}JEmkk_q7wag19~GB7E7WqD%0=7&M`ubvICSxYoUgglzS)0}{4c z;`TnL=o2M>0x62xz`8xX^ECEqXa%jm<0&iW3q+zN$L091^HH6cv9{UUO-8rz!%#iw z<(<$`+y~+GXz9D4(Sze<2Zd<1X-pmGEA@+eRcya3O~_M?uL|>+NwbFWxG0Ea6C&ld zZe5XI{Bn5_da@ziq%i?lR1~BpgXL3qNBXc5#DuB7M}(P}3JaB8EEI&59TwStZ{h`J zK1gUH-`b{i2s>)BGkDZuty2rSq%pULW++an46Slp_A)3QX%v>d-nd`$a?c8oMo~?_ zuxpJ%QV0?Fuy)>ZVR)W1B$dpd#Ez(s75NHYK`lj|;q7sfVgjL`-iR>M5%r*J=Z|)j zW=NJXKAjW!!(qa*9kg>cp7Q4FF#{H2WC%^`7a-CMxde^1V8M$bXTXY|F<@C(LA(yfh<3qcBB zWC(=o+h4%)#b&{i%d@=LSmeh(w8oNuYv&xKkLf-ktdm*T|LtnLH|uzL-z6 z+Qbe1+qs*f^zzC2sPYGZPWR##x?4R%`vuxKyd=t&14m!@^M(@O(|nd|xRT{ON^8XK zsLvthgndCgetB-0bgq}I*yG^^&a3J#1nKqL4A>dQl~e|JqlVj&8mBr6G`;GeplWP0 zzG?gb+x4XUF;+@IG8U=I@^m8l%(cWrL214d%b=HZr+IAJx`Z;p`|HSfY!C$^|4ITq zH*|138Z3AgNng~V{~ne;2-5lrT@~IDY9wuR{pm_?(_gWiQv>Qa6?)G6z0SEafXjzpfqM4RiZCGCK% z?h)eW5C1_nN42n``T>6o5@0tC${98lv=AN?bbv5Qz@R9v; z1SRjV3=VQ!$K5Xrju=6yBbCj*#9+?vF4?8xM7ljT!hjosxB#m5w=3w9^(az(yX5<0 zx<7_^#xUTD)zLn+5PJY%4VV>DRDHP+z?OV;)Zl}W7=HIR1A3kB$2cG zrE?BnMID4{yP1=S9GIEV*aScaB)-2iAjvW3aWZGi9R~Nd|1wT-Oug!x>B*^l`NZIs z>%X#kD;lv1Rs$VjYq7Gj1+q|^>G2%10=viuT}=%XGcT6bwdeo@#6}#vSK)Ynz`T2o zQ*oSp!poM?>a}}C>c5nNKvA|#KxF(y8RZb1zOQ-4E-%R!f^bpfvHB9eL%>5v${qiZ zq_FIw_w*ux`Jd#%50_PhlzgJO*Jxt@dPw^cEuT+MNzFIF@&Y~U8U}Q{-exBj&>ebT z7na6~l6J|b3%Qdb5SCJ4ADrBjd3ekDcSjloI%}J^Zq`yIa{qmJ!Qae8y&ol`_u)DR zoWB!)a+p@w%r>$vm4c2cEVbdOylnkUV{u=zb8G*Tt3W>v=XGM44*yVmeLdGF&tJYF zlxjm|#nI1~fY0NcT9zN=kqrh>mmdnj>$Ek*Lf}m3)iikdF$sqBb+(#AsQM^F*oTg^ zVoxLbH&PEBIkAx28E+CV@)TYhQYU#buq5&afa=ZTv-kdPM`SJH>f2FGgO(<@o|HgH zQ9XG^zIdD}N|pxIhL#}-E8A8cZ=(2N>vo@N4M234Glp^X4%^8)E6d{}FNo(SNnvAj zSUg~+wTvGR47TxbiXyU_#F}bud_NTTxW0|c-^x6a3?auIuWork#=5%H;l$ZJCv{D> zpnm7#Jc*~(An%1kT-cU)zxA&OzI*{xsSgNvj(hg(- zd$(TS_!tRxphYx4`-e+Bkxu^1NymPI2(J6hy`2|Le>T2o9s>QgWM&%j$(xP_CQ8Dm z>Mnt3cbtd$F`-XJeHq6~ z>w@0(3s0s$VGOsQ1;CjaG-*e*0#SUD^Dh_ffB5|7g08X4eJ9|94+Q01hLd^aOFdh! z!L1k+97}fla{2oaWC_`S2OuH=Jce?MS*U!=7YA2%euL-&j|nP;`#WwYL@Wd%ph^+e z`idYF7o&6=Ja&_1Sy;L5$|56}Wt4C%!AptpDDPy*LrSWI>uG?{*K6=YG>K-=*jCK9 ziU~vHR!)f6d%gLzM9~JXKXi0E_DT){E&YwjfqogA?P^Rw_Zu^Dnlytt9_sXGrPM<^ z7{3@$3+?EqPRnck5omvvmT%f{KmYewU-Uu$OD8Y6PxrLH0hHsd%oTGRnkXfeWkgp! zL#khG?}yRUp`Ct$X8rH$Bi3KuU~kLg_H&9Q$&@_&y=ak2@c4#xO{6f{2>4$Ys54GNnD$OLH#PI>KZ!`0o7*)tus(Vb#kkjP?W6g_ zext61NOZd&=2QYozG`XN$VLtKv!D9+C_*Hp-;$tX)tQj3_fhUzM&#pUnqYW@a|(*D zg+^ihBerzJp!gNZ`J7jL378H{oUfxU9bO9%zf*WKP{j0`9;M8ppIFm#vtW;QhJ*l>d;(xmZn+$%RVzV&Ug(;B8UkLK^H6#rwJB{(!bN2)E2Ulgh03Yf05 zbnzw@Jrar`JviY)&LB!~qj?mh1Y_>FTt`G8!*q;$xy46)hp{mstF_x70&)UdILXCq zCj3}y6MnW_xS~w2gi#3|H8+s%bPpK>+KcIC;Qn$XQ!^@9H3jN3*r+eVK0JCQL$u-3H=D z-;r&j6-Y7OlnUqM+q_t+hV%@F8GMYm`Xv#ozV`;}&D=!MpD2c4HCx3G!_8<(8>rr< z6Ga|prj0?WkFn8nGam727_od%LsO52!GqfNKSr_+v58fnj`Vbg zBC&jX5g_?SA{r%7im%@Bm(^H3$MPteAS6oi!m5@~`|hAsKNs}UGzgrVb5oK&HTO|I z6RS41;}eOXq?W=}zH1Zj1FSHsF=cwyqEx^5G4QHLeqi_LV z+m@=?kyYq*02~0{vx&Smk@*f+n9bDmcHo($)mAPiqN`0Yf;jt@;a}Ggc>07-BOJ_A z3cPRPdjNKN&d5Bj_}-IVb=CpkWyWAz3p-WYZ!%kyQq5?lCmPtgwbv|+=FAOie#OX$ z@e6BR@b}~Q%fA&n8@n6wVt9Fsk6i$9BVe(&tzW%(uO`8^# zBpadH`z2L4l>s?tofrREU3(1L=U_q&Ns3%^-Z?mv<#P&FLf6-q$Di_nvv82aNwz-0 z*d@k}Z_*hirj^X#R-JJ}_wxl2J;)NpZ4+WM7J=~H2$czRziy}K&tUUEXP^=CyW~M@ zjuJ*lUT;QHXDfLTao@j!!Wmy~Jv*z+zb%rShQ;cbupSTUE7ssfGS1C8Ns#kNa7HvZ z@4UnfNh?puFAa)?@ix7$%`<|^p}H$}->qBg;8=jDYkXeU(UmNFiY3W5vl|^5 zo~*R?Nu-X;m8Pr>2=fI@+;)V?zLUR*0|Yfds3|en6x<*nd4X1~r{!_lS1(DtDWp#9 zwm!`f-C2Us5W2m<;a$<#FcHBPAU(FS?B$7__6#5#kFb2ys^WI@YI_XLsf(ycdloX> z=C#2$wk=Kpz;QD+jxJ0gYL-zl$@&vTE~t0rWN8>ils~tDVM_!X3l~w+Y*L+fcf2o5 zuRbp`na|}K-dDUA*squc%~6P|-*_XWpKXsy&ATmpq-ZJ-j-A&NhM`!l0;sEIle5Bh zZyJ5N`}&&HOHyndSMVqpll-t?*LZrmm?h*6^OtsI3JDvnELro2c2{V#i-bP=po;HU) z84$<>dSTg06s~c`0jPsrYqH0}GRayJn2$ka7Ru+ZyU}0QzE;KhZ07hegq2+6J;1!} zbttp-iwT7FjW$Wb{a$T}xbzcxdHAG=QSWtNzg6d@&-ut@d{MWd{(~U?9OwCQS zi^<|K$oPFaDPis#m0-vy$NMTG*!4qd8S&2*2F8-YH+fg`2-V%CF?DNI>z0FEs17=O zf8|e6^2n6Z0;j$u$>^^<|NGdy{8ak~+ID#hXHT#6&5ju}rY;+-$&lgGVd@5Y^Lm0u zjkKh=+SojtwP6El!89YX5epRX9bPGf6t6Ku#HDSSGCL(~u4lmRzv+cQv*}6JC`Y2$ z+twj7;iR}oD0ZR$huIus8CR}f9iL>Qe<%{ zCe(7|$Yp7*y5;D?hZvz!l*-IC`w|^05J0ee?nLD2m`J=tF_HkHB~bEWlH58GDC^V& z#t*et6KxYuT5xMt=l_P$Wa9-&re&gWj3oiAj&Mqrgm5}e<*4lo4L1<2EXf@g9am&W zz*62<^X|ILc_1ei1CH+*2i;0H*80|#<4de6S}c$QL^<6>a9Ne%m7q&IFw}oG}$1)vB%(52IZ#gXWrx4Mg!zzw(SCo{fFXZH z_OJPTHa)gQ^F*g2>|}#?;l7wGNf&Q5pFveA_9cGKG;bL2 zvSJXZBW4KBGEBQLvXRmd(5?BeC=nbR%x@k9W`)(VX_p$CRpU;Ecs@66G2VE`!yrs( zZARG`mPoPn+aDOg`>bprX*60Go>@mvQY$?JxmGZEHlSN=^1Ch^6C~6n+`p8c5qy!j z_FmPm;YP{@-Gk6gwlAatTB}M=)8U(+s6B4|RVU#UDOA>k%E|MVX=0rH>VapgF*RQk zul2&rfCJ<+Lm|4w4~3Zr7s$FqPJQy<*_Y;vyr=+45CM}jgqI7M6ajJekthIZ^vVc- zxSu`sI_tmdz{Na^%s)$1&zvqUdCl{hr`mUOrbLw>*QHLXPzgca7Dc+TGiC>{Q->eF zC$9xtlrDs$j!Cl?|Bk6=`n}X0j-&)en)LV6QR^exg~>5li8(xS4jgl@QKl<<2DD?i zux8!WoyJ@H;5JX42#DJEeXtGKnZw1N8e(8n5ku%; zsFE#RKpdmr;wMJVMbPi57K-QPrT$}n*3i%8+n$+%iGTCQ_mR~MP{F@Pb zmdCo{F8?xV-GA!9&UBT7(0pJ5J^&G*C7<27GG18UjWa|$Y-dxRg{`{^d38Xabm)KR z-_XGJ*neDs&53d>MpT7iDeuX)M-B^x2Mt3jKWJ&=c||!2mWqo&So4wI^<1mG0K^4j z)jw>?XFCKnVe=~eZaCTr$H;Y;`*H-`hBEwoV?}@@;E{!clPDAl00xxc9?s;W;~GeI z3E{_AKLX(~g{%beeyNrwr0&!mfgccjaUFbT%cq*$pdvs(tFNOc_fra0raxi)nDBvo ze9ojm0`sM~r(fjW#hCVHNE!l7#n$Fzt_0XyK$W^=v?3Qv>eHfo?+kq|^U{qnMXa$k zRk5g$bZuzTjtVcsEPT<+HUKKY;mQdzyUWC>#^>yEfVE4;k(dMI{#@b1aHBp5SjA<; z5eV@#nVl2DL|TdfJ@88)&xQdqG-2)MVn9aHsPbJIIYkpKlLEfaWS#sRPdQ|?4<&4h z?AtVM&YoOK)-dKA4jn_*M(bd=y82%=N2=2f^|OIrN*ukkXzd#C+5)c^p_8R&334zc zs{W8HFf&9K%sngv&ur4(uXnGxr^U-6e#ejP2>Kms_yWzhQgIH72L`sM2V#90Bm>Qj z94gTO@5nd2Ji>rTmWkr;a;GECfb&fRXK|Mta=^x!m(HHP;s~vq8 zX%886G4P?P7Fw-0$WI1rHw2N>aiE-$9d*N?#pMt<(51vwV8pZ0?6v&864${Eo7u=r zhA%7moRje1;x$Floj2fZB6u>pFucIJEm@OEAm%FaYR((ptn{`oVhA|Bu(2s7&E%%L zdAC%eQ9_!%A8jv3q@TNXpG`(X8PJ=Fs@GTz^sXg`qya?OZ0XdHO9icqP+0&nZcwZo znb#XNw*o}*jpuR(`^(p%H7XbsP1&S(gT5T%_<<#(aps-VpHP+^&xpgpGt>Fn6sFwF z1ddJF11kR}M(2mWY5%)a*YvBWutXCUSQJLcbDA#j#fMGeDd}o3B|mo$Z5E?LJdJhV zet+nz0g8WMg9@Tv(r`nvbG}d4iD}%8PGeWnE=_A`43{M|>lVlDh}pZ)1O2AZVdiX` zWyi7q5@v5e0(!O253Za=`H>v6r@m=qx>$GQZ&P(|y!`P5nr4Zm|FrG&;-bZYGDH>+ zzykMj@zZP}@hUWSZB!2Vu8ZZycsloVK&S?3xiUxYSz*pKD)s>HRH{yhFOvv|=>g)- z-jn!=Z?^n){^i}hH9&F%SZK?0!1e>*c!Yf zC7~mE=nd`Vy>j#LV0M!mm^q^JON=(P7{(9_l1{B8I&$J&M@aR%Kvgv`acmRr7D{$I zzYz+za}6c&qrumFv02OPXC0N)v6s$_sj__C<~$icOG13nUMU3xC2q>c6v?j@NhiUw z-BT||u!7z3rvU+YV!N|GT(c0w?wCo};g$veyum8NmUt>uMpcr`UC+{>eXzaOPkcq= zEOkS8OYu!i>^ke?jRaG~gvwie1TA7)cgMi01gC7O*mA*d>zRPn9zczdoCG2YQ7Mae zuX;0MGuYnZ{>h^b2U!s%Us}*{xF>a}{xc$l?tZst0TFB47hvav1>t@0i2V%k!i;c? z!Gah1jsZJCeU^c0BZK`Ze6=bTYxh2pz-N9hIP&oF4*Z$bQxdkT+7vyOn{NP=y8DaG zLN(5Ad~+kaEd4Q*l^PY2GTj`ZE)~FKPn5TlnT`{fyAtsWWOY1N0*4MsCBM662S?23 z&O-z-jyS}|Cb%I9_jmPv6k0v!$vb(=!Elww`Msr6E7?maJFFtB&yc_G1Ov>U`Y(J) z5VfPts_Kw?JAnqOtT0a-Qdzs7<4_(=f=p#&21Xq-Ys2c$JBDY;rrKG`6^axSS)}=& z$qFQ`skDH9)^e?ggWOyFAnJkm;{0}+__cg!Bbt&H{gS5lN2c^43}+zoH2+PR^DYMjf5}uB|7Tk8aR8C|ql^^m-bx48t5>E?kKz|S`L90frG-uj z{k#d}Q7o$5xlN;=gw&BKM(`jcOvkDcz^~{1svwtz*YJ3RdF@?LRzog&;)o?k=34*V z^Z_F6Yg(&BLZn#*RTzhZ+>dHgj3@JQe?efPaDwZT@+F!bSi9uYrXNVjip2fllm+oC zWaKHdYuWmSSx{Enisb;twIVYQ&$Xb!jcoB}t0g79ufGigLge9eHjkQC4FO#<7X?9- z3e+%~(vY$?gru#6LwzzxUZrn-lu($7`rA~N7DR{|dK`0DWH(+7(NiLsr%OyXw;qer2Zl#QYfA zBG7^5CVBVYN^~IrKG&}cWGXv%)um(`%W932Znt|c(Rvnvl=Od!f<`--5yR_zD#fu| zGWj;cOikav+%2N1-gwVtbRlmlSM==Rk4u!W>UvStXH`V3AKetF{4BeBuh(_JZ1q%d zi6h-8DdCIK47SoeK=a%cJM)mZ`RmsO>gpXE28o6vhnQ_;03$%$zcEI`P0TyR>i6?Q z63uVmT@4Y8m>7=bw8qdkZU1ZiBH}k3iA^ul3LK1HjGL0O3_c zEeFZ#{T~1)K-jYfHjWL5Bti_RAZrRJYIRqE&NE zti6!LbtHUgS5A<{@GEhSEHuPy(`tFZRbeEv5r?{u{w(awDND<(<8p z3iadW!qnEt_Jb0INzJPtjYXfwHu(F02)!3k6}*0BfpbUK>lD9d7e2ORspd1tgxM7c zp6=GJc2mu8$VG`#i5-bSB39kwf4f2c&G?~(XBAS{OFZoqD_qfo05iZ2iV#q1)M2q} z$>n*9OId2{Dus!5Ia73tutV#{`k;(a$}V3~CHw^_&U4In5G0SBLQB?ipa~c+n~t*-<`qWH?{%&4S{hUbXl*EJ*1|@)%M>yTx*l zXQf|eeii}t81)^zxVQMFm5gmj4@~{yOdL0FHYxXXatB(nwS*!ur9p5~ zv!phE;_GuDJ^nA~#t*hF#Jt2PtQV9Xo7Ltw_@ITTmz|1#|3|dSOH??`TbqcF5IRG8o{3Ia`3=76f0tF zD*9=>s3l;KLR6-)8ng!>iAAdxsJf6*YIz{yGoZ_r6IN<|Z(EfXeV!f56}zEC07Z}} z*4io}T|IB@f`ZVWHM$K+D(j(H5F2!OM$?t#14^YuRP_X8cjl7c&+hXAh1uk#1Bm!T z+PYwwie4=x^QF>4+m>WZxAgJgTkOZAV@|QvHPm$fr<);Lgm3DsQ^hMb_~mfNI)C6B z=Sp7o_X}eZjWgbFh?n%J@5+`Ynm;=s1UxjU2Im}F`4`ncy`jHT4QA<@G@1LRl(mp z=l44o4o25|tH(dy9Wg-|pkOJ@<$$ns#-~mk z<)JkZvgrso8s7XTY!Kr-T8SSUhY?G^w#(07gNbEb9AAYDOKGJH6CA)WOF1D}Bkn(0 zTa}Y1%}m8dslst*fTTz~q(8i$%0Gax0=W3lNdkppJPScm30)?Hs9gLYR?ys{jB3UN z8Jsm%_r-PfypTc8GMbQ}3zMTjcp8)nuXWpgg9wH%&8X43rFBv7XeZrzxqi;zIs$P2 zNq9K6QII+>%98Sr<6#TW7VHL4w|4MY$jP7d0LwZ`_yVx@N~>slOQw`{E_enevu)d}ck>KRG zBKU%@UdsXB`Ik~4t5%SrJFb}6stg<$z%yCfMQ=@`a9QZ#II80bJWFRYCW)fLz{HqV zD+HXqQNvz9_<|lS{)=54`t7X{qk+K_JOVz@=Hp$kOY_^cw zNBc#c*d&Bo4vBoI>qe~>qZna-oDGfWuEb+9K-pwrhCX(fjJtMI3v1LrOoTXYeYb(& zVCh$JaRM%Nj8z+c-%K9Uxv$0<)M!+?_+vr{b^Z}9It35H%7c6EbHcyEk9Cl!sw)mE zo#xm0Y)UauodiA%gtH;7Pj`zK&`PA5xcYYTlCl`iSrcAa?xlmvb6sP|=?{F;eAe&9 z3gtY;b&%yu)6pRIt4Rt>dsNHQ`h|9>)Jsf$&0hAd#fU!ohk5HkWu-4O!rU+! z4)#7KDk>`f<(Y8#Ui%xDcK5wygaN>tV; z6iQVBfD6!#K+Q^DdB$rC(yo7JJLEA0EsNxUGe8dQ2<^g)Qpu;t>l>=^8oM*=DLt#U9JD7Bi;)}wQSa#R%CJii<@;pRY)@Vbdh&$ejXl<()-TNRkOlaWaK8@i zzhmQV&|VMK0qFtuk`J|__WXBzweZBSU+UTScqgff&WyL4(oC%|ME|oClYj15K!50p z?2OG9h=siIKWE;c;FNl{5XB{m@04dk5Fp(%qD6z6)cM9QaUs!PtKQJtKwB*ptOYq+3b1r6FFsEMG1h4T?K)4E3|cE$fGjK4TtWh4zCoA7x~bA z*cO%G+Mg2OSZO*B^iShSj1)kB?t*k^WrmF{JRViBE39*xF$lVm(NyRpp}wWwo%0_YW#DrWwz{ z_-H?pl-8d)s2&6?EMR+^iQPpoFs# zc;(ejuxn=s1PwBFc;aahr)q8%mNVTd+-PKtRFXQ99-AM=w|p&@XGV*f!-ky}Sm=V155sKwkd<$WUt5;CPZr+1n*9Qd} z0Xz~@sa6=v3fqUwPRj$0t@70u`K;%_?lq#UA79ErD0#?~#UG=uV;M_V?#tf)0kueI zTiJPhntKQxPq0N&QmftnByfTLys$IX6HK9Jq|V{ubjsZ!GtIDQQzS<sZiE#-Gs+SFpCsm+ReROlu}`Ah$n#r23akYS~s0)u!Rk7xZ2=Cn8N z%Fg-T7b1g}Q;8R{xT4VAOCPy*tDt<*PN=3%|Kv2o+Arh{Ablr1=l<<;xXYv)jVG&T^10Z<_aEL>`GDQDO654F6_ zKSImVIXK6yNxSgoQLgeAYyiyQrGBo!lL=$Od_y)JPrqrZ3FP9V3W@ZpG`3w=YGUp8 z0p)IPY`Qw^3-JE)5UjuG24@rvbQ(U?RQAUITT?k#&lX%grgoEe(T<~P_L=w?uI>_vvR@7mpdPMJP6}X zUD=^xj(ybJwEIpDz)8T=m zsE{oW#QlOFIkBb?40~r`jBMje<;a7(wICs|h=-y~Ga-Nyr(bl$s*X22TTiUP`(*#L z?wY>*hUkDu>|?rjiPb7fIBE^zeTT)#I}`cQwGeyhgng;0S3zl?Q`ZY{13dk06j#Nd zN~|36O&y$Gq7}W5kkmk4_Azft2FcTM8KTr!#3Rz*2PTqsUMhjDQx3-7(RLYRjc=#D zz-Ofkk+OxN?kbh*WVGE&Ix6JCpk|-(S8U zHyKe)qX=ns^Zk+w5nFJPn?tnE(ORNo#aS)&L5YtcCt+HvRn{?MB41}s=zMQ`ge(Jd zLPVK;fY|c+DwK!iZ*OReBUfPBce&9wHF-#*g2)5fGx(CGy?kRd&d79E?$en68zTvu zvAJq;YtrpK$h}x%o4MV3h~*WdXc4oC<7D)Li4c#Ed-?+aFMn|57dZhg45)+@X0gIi zH7!T3l7ZtrM!HSA=v^%$rd<+ErYWuLR|Ao96+(o%IXINcAC!=4k1LM}|Ma5!JekBf z8}L|}v~s_iMc=)A%KiBC_PPKEs${m7GxCxWDRA$n!yv&?l_;23PFbcW7$`|)h%*(_ zD^25CjjNR{K9wqY%ax1IFk{%1Ls(Hi0oqggmw-ZJF}pUeA47DYa(yY1(MLmmLTLGD z6iJlfP~rRy&ssh5-@+m0h6+T9YRgDuSBF&g`Na%F{(67GH0ISRHm?z5Wsu^@R}-R7FdGk$aQWEA!?cI3olq`k4+s3Z9Q-x51ZPhP zc_9-qI9p-wm=+j#b(!sY826dycR}rZ<^p7xbr318zJC0xStcnGyz3>0;nh;a33Lah z(maU^DJ87Z?6WDXnPE|Fh+hx~RiXhFTn(51}%-n~!y?Nn*6lCPI)b2p%cjeMMm9Rb>j4HMJJ2loA3Yvn8lFZv2ME%o^6^)jiWGq0WgP16 z#x|F?O|1ls3|zlen_CoFsy{_F>W|#SkE=L*KGOHw(*@o)6Ch4C@!}R^Q-8K;NMn@w zini6M4R7#l(CX9> zrCIr6Ri;rr#4YTRl_G96u&&ap%6qRu9MkOsEET#4p`*n~Uen3F$YvoaG-u{t4x3kj z$CP}yp~?wI-)fh(EzmZ3D5dsHC%2^WNC<8+?fr$I+0_M_G7-ylhl0ZmeOrXg_zdUo z`?gQPGZh{<-LAxmQ3by+7}c4)&Tbzybo<9u=MqibJgeB=*Z~?)!`UQtti_w^e~LB* z``y~>rjhn8a6N_LzG=ly#=Et~foIx{6?}-Prw8_~=G6D=o5cX8_|F$z?nrYY1?{P95%JMUjvkN1{UhvT;PHb5QgDtnk-3_r3>O>TWQEetL zApbj^J_o60!C24NTk-$B;pY`sqQ ze6g@yA?j8LE`^T$Cl!m0`}jkWU)uC0piCgq6B5LlbAGt>Jg5t*H{GUC*E&w%1_F#^qQ=#M82)sL{H<6+$F7q&Us z%WJ45^qlJwGAEW3P*a+VXgojTqZDUbls=Grm}ffvA9AiP5+S$!#7zV6aIM?JLs`Po zq+?B#@$@*3ZjcVR#_=^k(2HBK=wXm?69A6YzC&$zy04E7XO_i563RsA+;r><=V0`y zV7~%(z?WglW}QA&TaznJUSElDMP!F%6eTH#b?A)O<6U55izWN>=l8IZ@y0}(Sutbg zU%=d3@$_wzC>XtUq>Y7RtdX^=Ae!N+M<}Nik0N+M+oS}kNbq#faG>BHu{SbpAKTQp z`CD;=2k*V#Ej-nK#oQaydVVwuz5{oT%JPWHos$FM<@u6$f<}>iuD7~D2a&uikI-TdUiq8YpfQ)&sxwUPq=pCKTTg)O(jOnCG zDE`z_mhyO_Z8u%yW+vdJOs5~McUcsMf_CfPyi`0Pw3{J@-7}4Gfd}GFf=o_1+*1B~ z-#do&S$Wi(4R~@S>DhA4_Rcyo6QG2T$(tFa2UV{Uvac~zvVl~ThNvQJNjNQ0SSYr0 zOU!I>mPo>x3eT;09_{eJTQ+Hn@J5F~nn|fWo`zGkEq@tef2IxU)B< zIUU!zzj`y=bQQ$$T^`LIAWSk;u=-T+Ix6;;hMUau^0|GDdD|U(PH5L&*P3BDh0%%i zUHe#4uY`cp@MB7AZnJ~OVkda28_PByiO&pxc!F=?RL#mI@Wm++RQg`%BY4j#g7ufX4N6hvs zjbz!@Bs@JVmsz#z)RINPa40v(Rt{3wSgr+9bQHQ1Ej=S4pJG!=-BhTnxNip;A~Q&E zK(c=3l_`|t8*Dzni&9mk(e2{lyf#O?#~ry4K^!FPdVV~nZb6lG+ff&wuxf;d7Or+@ zf!*3-;qQUi2<5bumS#IV2|}@wdm3eUOGn#-6b_%Z<=Ez;V}T5>J1d!fQHj}8ZSA@? zq&SZt?<}ndDjLD4m@EH7hZ2QmD$HoH{-_%X(*6HCycU&nn4eB6b*fY6>U$}UTnzT| z3)%Hy=~e?wK9gAM($#T4>uFjEC>kR~=%tk@#%2Tq<8J(?SDj(QtzSy0 z?fAS4XFvm<*3=_z`H^7-SZq5;eClB7)pSSijE>MI6cZc#@lDbi>f>QVkfc9;Dw zCBk#HL3_q3x9dr3KLa`;4W=&pmuGz>8~QVh#sIDJ$y7QF-2_qT^|n{=aeo$jnMJ zsfNbN)5R7u?!~Ckdmyavv8UOMR-BR0{ytb7_iKRAxirrvBWt8@@dNmaUQ7?7& z+OaaQ0rhttrak|inB*izsMLmAz1aT@=87zPR2MJZ ze>az$7CEc{KstA*He~{wzq$u<;E)zkeoHNp_NI}@b9Yc*I$qHm4J0B2WF_!-b)5z$ zrB1d#DW%3aIbCoEIMhRj>AI9T#9FG;F>Q8@Q_?Nau5(2^-iQFGwdar@YHjrsc^ysf zl20$mmF|olEwBF5SL{Nmlco+lMkQ3_%mppk+Bs@ymmcy%zi4lf5$M_(U|9VlXLdovx;p>{6db%-_SGgf7!Uve2 zqGS&V8r0D(AO= z@9l!-*>m!>;tz2evX?&DqwmsQviV96mRMGVN}pKbF4DY>_lv5t7c*F&n->pLo@o^_ z^eiVAQ8ej~{NJ-3ede6jS9}+hoDEc_T`HM}fu1@L(D9DrU|)7m3#MJGn{JWoH|%w7 zj$7NwIz*&qd_+sAOhePjJW(RgOaSeF;_c|f4*^XHvjmuwk9hNiI*Umw5#f#ml5jj= zmvb%u%Xv*N{xDJ@;zjQcOBP~ejaZt0;r`(Qgi1|Wc0`xEsHz`3IQZ^D-Y;mBBYaDw zZjFH4$&K4~R_)gl457R(@mS{4;gE)d4#KU*f`ejfOFjWaAHfaO9x5xX80Mj9(}b&> z%)ED2N}#cWPdgR1FUG#+jlH&Cx(Tyfhd{U%?3?!TgM&wSn~;RcoVb%{)#XKBcR&(q zz02*^0VMR=uE+Jd>Xj>{KG=19HJv`~nhD?*vZDn3*Bw$y-%W-_(F$qGRp?*od6?A$ zWGBDc5hGk?&KgP|k`2?-xK7ddZuX9ALhY3d5|}S05;Ymutn5b^|4S4fN4#8$r^8&k zMT_IAs@elsx0n_>$MX#>$RP?Yu1#TjqR0gf{@WcN??X@WVYcK3pSOT=&=b?r!hjCy zJ8v8$?fTdiBEdmG%iafJ3gg%+MjAQuI*25FaiHP!g=2)?Yc|BPjJSsW?OrJwIkcS@ z8fQU)+9Fjg?&pHQMH-EC2a|0rAR@!68Gnlb(I-Mt#^@l*$U&IAULt^9O)WSw7Uwkz zMTR`y3&6GEfqN-U!OHokyf?mMAk#NugXrYiOB*r^4ITKiB!@yWB(uGGe)=_%;dg+VTf!l}r&#d+gQ)`*%7YD+~jZ1sBKv1AVjo0sK_ z$7pF{i#3YSm7GP1WY=U9UUs)owh>>yg>B&xMwd8feb(QXWL&>f*NEcm|5vgvplWhFUz9N=c597RFf%jaVgK4TZ3XOd)Kqizp zH!l_2?y>}HG#cO0NB?B#3B?AiIxH~}wfw_``zN0cE3qoe$3vEag+z`UhfCK>4SZ=$ zO#n05H40&}zCPpP?W5`<4sM=e#Za_8sDlLL&d%+Ge@9geiC z^6B?Te4fy7p0w<2(cZDT{q&G|Mrdr?05tqZV!BIqUDjxd?8in|0Z?S4MzRNLzPSBfdNA zEg)d(;&ACmOi78OoQ}n-;Dz>|&F4fv%Z9gN9ZOa12L?KjV0^s@rT3nozD(UUohvxRD@9sa{&X5fO3m569Q%~i;xQOwlGV*DghZU2#=TB80^D% zbOTQVby-;LdlG-kyo_=AZgpTd93V0+A>#&{%zSAV+}{EjZuC*Cb2nO4c+}T9{>pk& zyWr(UI0VY;&`1_N-RIG7+~Aq}-&eLyLe2j&?q zOlGAgC4XZjQQcEnaUp^(5+_c8!P3KURr+x*IY8=-vm;RIE(wJV1|*zr;%|p_0zyW^ z-=)ad^`q2I%}~7gVPSK4Un?!Fn2gdHblP z3g&5(Foh_qAj&oNrWFbB_=@c?ZKaWBy2Z(zS}}gPP$^9Ka|5!x1#OrM>J`zZ2yGYR>3gzuTB>EiAFIBPXa1Wn z77E0TnruVBG$cldRoM!S>ia+D_iQ~7bOXx)5V6y2QfHv8fWoQ5SaDH#?icZ<@D99j zQYtFzc!-N5megGG%?B0WBOcLcaRD~8L@kx=D+f-SRgGX{f0lPnQj0?f2#53zA3F4e z5J<-y3r`@EM`;n$XqlK2r$G3mI_0PeXVr!8ZY5X@25pV=T7z2W0&x%CRW;!8Q{ zh)f_lBoRsp5QJ$Op8SPTNY-dz4c@@pFg2W$uwc*-^5xQU9i}jO$bmFCGLzKR?E$WRALE)q?%f2KFjux4k-EfaWZhx>@tq7WIKJ>(Gq+ zNgL4bR=DC?%d`^&WY)_5E#|=&-EoloV}Hl=vIjJOyWSm*5<$wh9=m5W#l`PM2K5?m zQ6xf_6ELor%;R`T>BiP!;_^x*vE+C}L7f zPmpFUGWSJgbkRhuS-qRMC%xv%@amHY{QZ4aA*7lZ!;2*`!H2NKm2=80+b$who23T! zEUs@YfW)8#5^Y-US->MyL=wZj zIFZr~t}KKIipm67sl$uvihRc!f6cm4zYX^4=I_1d{@k{aVd8;iuDx0D9>XD7oOp2Q zsIQ~0Zf~+}(p?KN;BTlqj>4G}V%Gk{gqXiWNGFEI+KhyXT7SY{h*;}X;IM`it(Z}@ zq6p{MFLp*xWjjt7j?~fCE=b#j^_hi&q-(EbmIiEaA_UY^jmwpfOUA$Ys(dhF|0mrJ zUQ-`ov$2j(tLI!)@a}Jh0cV5fx5`9tKQ=Nmb-Gk4WxNg<&Z&cbR9`y!IMm1yz6K-^ zVyl6FkJ2sA5!Ymb{~i7=N$nyw0E>vyg&)8{afs5RP6(7$1w35FbLsybayi=_SY?Q4 zWu6=}aSH1||3JE|g+R`rrZS2(CmZCNNjX%k*=zISw}T*nTIqk&orco(!<82UcMChe zuKABJ0ff3R23JAYWH&57wyH+lNGAjCLo`I;R?61-Mh}-@(kE)0$y%v1<(b4GGe^2i>)ATlHQzx}Td`uEZD9$aNpP6kooo$3*FFyL;Rwqs5@W z*joty` zJ|>TCCTRo*q;7Zl@1iQdF#}PQ9aCP^-|f7$>@u-JiX|ysf}XUR(Jz5aN8R$V@=_li zQLuZS^g)HVv|O1*GB4d=AFz?F;a!G53;T>W z{r!|O-LjxyvX%GlA_*v5?aC(T_YEG990S$@jAN@`<%b%UOw#av8G6BWm;2j0H@wiy zc9Jd{0rkpBVmX@V<~(Tq^6ufmb4~wc(;oTd7}|Apid~YA{&>_Y=JL9RB6n6ssC93P zSz}d^=Ct@xZD(hXp3pMb*0@0BIwp!y>)(3TCR&!Ugag@^r&U*PI9&?T0zjZEG+&h+ zrF8r}2W}5l$VpJVzV$dxqG>&b@G3Zpfg5WVszwr{U0D$AhRQJaYwM0%~!0FFtkDID|V)5tBNRewNDg|S;8() z_rx^{pMqa#I8j%jD@!yGsb`Ft<-|asJ zQ(|>dQWP@ehc8lXJk#^@2P+OmQ<>`{BxdPW0?k-{-bdfily88h?8*?6-2=Vx$t_OZ&<`c!vXmiKG>@> z7QWQ4h_!f|{vg|&ep6syN14A&dV*QO|DG1hj=VONSwyzf1uhqI#>?sOe-cLHL&30= z2SUVvg@+~+z`i_v2G2X+tq#K-vY&cExDxfQ^^h#RQS&$VGLE9_@%@Yu(&a~C4Y@Bb zzwt68$kTp5=j)3OF9G(J%h@MSbP)W|F|yXQ5K@xHi#J{KObtRvY<5#Uagx zl+gyrgW2df&P*It4F+VKe34X%y>W`G%c^F(g|*0)$Yad)IoU7B7y;=*XMgnCt;&Y_ zIcm6p$sZ*SeyF^+KvvO@LbQWE{}2g5Fr~h_!%`icW^eoII=av4RhjHMY--nA(R|l& zSO0HZBz}n~8PUzFw_0_e_7HI&KJelN({f4po*A5Tqjj#%mT4FaqVCD%z!}V(>(V-d zTA#Yam4?wj>0tqDq+xStaWlic=w!Yd=)b_UfAYqf&1?^ zyYedqME!)_{~2cpEA?#mS97H&yMxhcT2pc5qKzhLo)iz_MbJrP>I=UE&Lq*R-0N^7 zZhJVif?}B|KMlV7}r;rr&Ex$XS#Q`!@ac&P7bm8z!GG1I?x7%B8-!h==j& z$vTTE-wCYK51KxuvS;6^*usr;TX`=bdNwnxigk4sbg_$#5=5oXvM+iL9-C>T(7xNK zXa*&-xZK5>kd3&GQ2e;!qFTb4$`2CPA*6$`NzO$5N_`0Ue3@|5e?Og)vsZ#FkYj@S zaP1PqsWC*tB+3}M-&7Xo6-V_iKE;CO#GCL{Yo}hQcwvOGeN10{a_~JB@~<}3bh&=H zSY(AvIw1U$L||ToHnoV&?{KuWe4Y7KXG#~skuK#y)7lS10&_mos{}}M!!V_GyX?nV z4HIDx;#H76nt|jC>E#cX2h9%pTA2^QpD1>;+8A7$O|~~i=&?xUU%wf*qJm5}(qaoo z=*?uh7mQA_yBg)dm{g^wCyx<^eVX)Ez8@wIrb2(>la7~_ z!d4I)X-e-}bZUQz{P=T{(#)C`u(On#Jr^0#ZPg_za43 z^CWgpP#J3qE6%&Ks3wHT%-D@Of}dWVz@t^H#{x%V1H@vjP3qIAKfr!4@QY?c@HNc% z$HQN>;1w1+J9j7K7>MGQ|MielLt^sel46kZ#HsLd&S&Ulc;p4(TwIf}2vwZxaze&J zByDsuXc<4X>MBCf%a!~P{7Wf|MdL7^A>&){w%Da$D8S6-?d!(GyZ2a`OkzIXe0ipc zjxxDqA6}mkebiRmfKDSujJSUis(Qd{VK)CcAfo1{6up?{m;2^aV~6;9?Ngf2EByKR zL#nvJa`I!m&v}@8KVCQ8-fHrb#{}TU9ximm&W`cJ zH}I_srk5NAlohBZ?h`~_#?#*m!Qkh%8LYcRc?0u6ETN-}v=Ik|KN`Kq|8|=kO*)gC zS~I5qyE32w>im2Q;`e=d_E}z<&LBeoa%rfmNwkQ!zv#I&oo~{LBeiNCT=qjqsMvIy zyKFKxaMukUvFe0S$0O^7SH6D%Wp`o144s$@6NRbPH~4~=XS*|IAeogn%Fbf4oH^6V zfi~D6@EYK0VaQ6NNll7HjdT|pyxC;@Tmo54ziVcxv1{|@CO}UW(<*)Q!qHe(s}|BUIE-R zge5Lt5+%ZhHiRf+OAeCk=MsL=^$C7+abCvwyKtK*W~1WdrO427`^VRnQ)_5gru>#T zFuU%un=o-rNOZDAYp!Hp45nOP^#DZ$HMnr!*7!eBDLhHR^1QlJ6g>X4W)M$4m{8qe z+CG)DClTLiK$)6FLf%;v%;M`7hJ7`tfYQb%#>6W~Qbx7#vN_~WyG0%kmN1)$HW)(gY(rj0|LKz7F-o0L3i z{zg&*^rvPiSbAyHyPhXh697BiVHv9ZKOR2Bt2z>Zl=60frTH8b{b_$la@6VzvtY}j zpPyQ1SNim1_=JZzU6+jr!Wp0ylh*381-U}=l(XE5+mH<@V6}qcu8Gm}jW^rmB6E5V zvj56-jr0zY`Szy%MQ&c^eNPbZH)NIG++CNkzW_UFi@LhZkkazzwwK$SPGmKVz6d zV-1~kyuCYmctsG)82vRSr@`G|e;w$xKQg5IfaQj#LTBG}F*J1tpec*CAO^Hm0dbeT zY=;_&0SUC;e&8VQEl~K~45+UbNEu7m=9t74O=7o+l#za>?a)8`argg-IeqnMe=k@N zO{I6Rt%~-xDcLA7X#_A_!4z^2iS&W%yteECH_~3&@A{NJxD)ahQmxnyLG5+ z1-t-|jJB2eFk-?XLhAG$dA-Ag+16)#!zIc~-NOsyy}#Ir&jwXi9xoP-xATRvH_-nq0 zPOSr)p`YHcPe2rp-0WX@JM$NQRe||>C${YD;kNe&r5qz|GN9dMi4iRHwd8oDP}9(g z(je*SOPs*4QQOJXXb+L;jX{T&3{^KUz2w$!)+Z8!PxtxfaT6*Y5dSsWmYxT|BF%K8 zYdnMNThn4)a42wKJI#)nze2&(M!P;J>u))=0Pho#T8+@nME7L%)N5~r@jfO}4Roy+ z-`wpoL4%-iYXGuUD3t^9fv+Fu^h>#8pN1Ak2hyNJVCcMtvVY)TI^&;$*)#oC=;5JD z<~H<3G%^*uwcz**U zJw$j>)hpzqFTmaPDZ&naq_1m1`P#PAafKV#(6RqD!SwAiH-)?`*g2hAcH|S5WZ;ItUA%-m$>)U%x%+!@)euBIJmzQc ztI5oHqC-{>m;iYRyZQ2}K2(Y7M7peb;1kErG>;lkjR@mgms=F)Vc|ddDkDZ=DC{u7 z5(3VssrSU!fKP>S7#8I{x}lwOpwPQ?tqGts`7Gsz3V`Y-Fv$dYS?$zECGygf7B_AS zYQ?j@)9Ybgln%OG=}jN<6!zdE!Q|q|jA@2c^KE z#h+9#*u{UJr9T|NiP}L6{e@2LqMWmMizWHQ>h>#zR9F~3bvdDp2)uPm--zcu$F_5v zD48@^G4p}W#tD^)T1DMt(awK-oxd$L3rSmEG_aN8j*A7*qY$3cgDiIvHPGAixXDuN z>L;V`eLS!GU!qQ<(Z->vySOyLgBOsQ=rF53cj~Zxhx-6v*zHssYo(!*Pj99bW*81j zDg39NnrrS^E}bo@b?>e*+x4JblW@|@Fp(0DxO%h{&6Q6B2yXj?<@ej%yWL2P5SQIK z=_ad&V?r?qRZZh$#o7^TpAPVWvndJSxVH;vH_+>q#E`eVey`zc~0iDKvays+@7) z8BV?mm~h=NieUTqxT>(qIH?JuOvyg09ESJ}05w3$zo0Q6`kdUTc0gj<7tdo4;>)*V z@)j69DD4M)Bs%6l#*CG++jTarLq;~f`+8{EzaPIIoyiF*BINN$wmW`)%pR!J`r|Xf zOTqF^Huiu3J3z$0Hw}v9pv?~VK}XR?03>6FU1Ug_=0rFAX{$5;) z5F4!`(5^~!Vquvey#y7jXb@OXXQIIeHeET7x|#eWT_Uh;#V+tygBB;|Jph%){p%y1 z4@00*p{sAWlb!Jg*{E~Nisw$9=*}wc5u~xmt(}!F>lcGC21^8f$ZoxR(x(ROWy(Wy zlZM*7f{OL5_{=_vHued#9EGYPhC|Y28s(-=s|fTRR4X-HJt!Ipj$je^BKY<-h z=`O!|X(E%FK8$(vqx{UsjxrnKkjN?7De&I-LhhhgPuje!@d>h=lCUB&D)BURMnlud za5rtHb{x>7OxlkPmZ(Ww(OLe##b$KLPIL8ON=)4Rf?2HraV93s9O1|#IXJ!b*JRxd zdE5BkUut;~99_?lQ0_B{oR;eT_6iJHdW0YjyeFH-}pg@2AS7Ohs2 zkReQnX)46ZluzM9VXDK9DBY7cw>9YWI$)}B@XQJE1>r@AJA-e3{=zQ>H|APp>c3)Q zyEM3~ExWr->mUhCNcnA@wa&}1ouXAM?Myti5aenX(RsdQbb_Fp2XLewYyjkQ?z$KI z82er|kWUq)xb_sOh@M`9cY4%pbFt|4x z`FB$}QsBH>tBI!+FOP{sM&Aq}sJi#dWtJohd`i6(<0dad06c)rTZ^>!Eg^5+Jv3zF zD@|vKw*~8*j>skEmc^%gPaWZ`V_i$2xR9igxfq`l-X?i7SIHTPk95B6j^$KI zC7gGR%b35Gr!(XekzVH{)aA9(pJYo55`pLkW=CAk5D@!qy>X8gku5P0>dS0=!d6D) zX?vj`kK)d$3&{H~UOb%`71i=@eW`as3y9?x_5B-n36Wne4&b)I?QS_j#36;DL@=Fp z0oFLtN1wR%!yiQHMtPo3Ae7$j!I0^L$?>l+ByeIOk$GVK=ToYm)M@#!I;st7?{Pg5 z@)GHPZz501*fDx-qIv-8JVFou%W%=ae7nsM68%>j-lg&k)2LYY#(oM4eQG^?1`N!k z1o$-)3(cLE>nau>MQUUz%+Y{ZZsx&jFDI}(E?N%0tq<>#UETR2o^=uNKoOK|xqT$hgofkccvuDtkkqR6RHLbYR_DFQSK0pUZFCGDAbsu`6O8;5a&lM}I_L?~o3wKQ zlQYjbVKqP5ljw!O`;+zB0cxiv4=hTK~4`zu(o$1f}rv$12-( z5vV=EioXRzAZrq{o{$XRMAc2S{Aut49jGoiqg>8e zbMB!M6ps8;D2=SydX*b9(v7^bi1;hUaHRqv38=(06u<|{>^O~>gG>&Pr)C3X;=}_Y z`zg>w2VjFMiyb~wD3U}MPNKs)s_^ac)~y<9bZb^FXb`pqSnr&10Rw*3OxuE+a`5SO zKPB$wc~laFEehP$>Ge5>)MHR^rB<0Bg-it9tr8qs`A!!kg0WTxe08ywg;qxL(1yH_ z&VzK#6FO!jwtvw1O@CP144=D}h}%)|G%5VgX@hBan7mB;b`ws!5Q_^umGGEU@(m9I znJswX+L^VjV-0$w##Syll*tdCa<;gXzhXj;(1#GM29d#uNvm|dKr6DbQ& z4RG){l=!LgY4kBpq8eN4+v$net2RM0i^2WDgNsVjddtH+z(~ksvPljuMv0#}8_LQs zP~Wjs9GIikVmfJI z0dsR4^>pNiV$8-b-4h-jwax>ojRlPDXr}D(;s%C}Py;?aX73r>sbqxDIL_9G$1ORR zO=19*nFjU6{B-Asg*k9gb0>OZU6utq==hs>W5IP}e^!>D_L4sl;=l7oZtl&?bq7K) z1{MleBemteKcvee_GJP<(0*wx(Qc8*=J4m?KxJmC!Td)Lzm8go_ds8F-29P+l0wJa z1GX`u6io(Lj{$`*A_{q$n{h0Z^#r8E+=3A4Q?Gv8Y2U_;?P_NFH-&-WacEW)SJ*eZ z%lG28a7##Znah08*B1dRd-qCLMY~&`(w%CSdpr>4(H_2H8@xO|sz6vFbt~c( zoL%gavbc2NQOLM|FJWDqQ`xvPUK{`^_G(?zqJ8F;ddkmw)y#Jr}={$NRan?nfBg57K;M_cmtfz!KwZLs44 zmGoa_H;{xNn6en4sj^R|w~BRkdB#?CorJ17gZ+k)KZQ{BPu73hX)9dI***TKoU6c5AGt0*gQ!!1#?7zWFOLm|x^!kg$m-SsJ!hLP?Fuz~i{p;s(+(jhN>7aM#(jb!`3LN5T@2v4jL zBX)jGj`PzKm zEVW}OF+7wa+Crn?2%RYWv`M%Y!bLhXjvvAaU3OrRZu!0D5z+Z*KY^e}sDBYSI7-6r_Q<5)4Z6zBuu29c??iBX=%^Vlcu*F zSZ!7h`-CO1RCIUPk&j$4zbixAwjC&#-)iZCQLBMYDraTuTRnRbwhRu!IulIif6O&c zd*s-}ckiZ&s+bZ1E2shT;U_jgL~D*&-Ujb1FA}w=i~U&9k9#Z8TK$Pvb7=&}(zq$V zL9pBd`N*GyklbN2VmoG#i-7=?620*D1vXc)7{h^=bIy+YI08V~|x9 z9Mi##REth}w$5ydFgjPi<{kqip2aR;pqJ)huazdxDYR09CXim~87!5m+SXaxCkpR_ zJeW%tx;zWUl}a=0$-kY&0W2=n0^CFV*z12#G-X zn05bTE=d`@rN1PgI+^x%3e#~bZNx@@tOgE|gGVx+aKL%q1ZRt=nS;@jg}p=`H7Mg{ z`1BxM9v&M|dJ#eG2ngqI7u>*jUkJ03Ittc2^2)4A71@KTG5lG)U*AX z89|apd_E7H&BV#fp%N;>{<0Mh{}1^^esVh5w$Hh#J+&?7R3^h_jaixPE%2~7RhSHS zWW~Ji`bvo)4pgTR;wEMD`YY}S+&wL8<;C%(hdJoloAj5*S*b`JxNeG z2BXt|q@VIpQ7Z>r1K*lHIRfNxY&Otyk#+;gro=xou`$*ewazetsc?6n28MGl$WY!* z{m2HEu6%+?ZzCDhSGTMA<11;J_)Q0hhy>4jK|4KDGC+g2p2Lirkj0Df$Ih4Sht^Y| zOnc9OE4#gn$m_P=WVc@fYKsL(^SXqo0Zwq7KQq$5z3XB?1!-9(LaY$MehFQW)GeI+ zZJ@yGL4ouaELyJ}po9wlX?J<975UMCpLgc~F{c~5MQEeoFk1S5*;u&Txr%loec}BP zpr#Yx4d0gP^we_$RTY0+n3>3XvBz&L}Cf7{x6 z%DEaw_~2{$Furinq{l=Y%^jNlwwRK1|L>(033)-_Yp!3}HyB<7L_Zi=Z14QN^Qx?x z{Hta-s^*t6MQ%Hl;s1kamp^Ss7OzNII`6B;r&{?NDeQtKD z?i`)yR{ExVVkuhK!H_liP91YD%euV-#Yp?&B&U`T!M4lN{hswWp`H6f!p4~_C{JM!{yWi6l8$nXG0D6*BsO>HJNiPa#8_i?nKsd<4HBDuaW5SWx zcy~b0G*t{%04}qrESX=8^v%_@KScLH`4~Gzuzfv0)PX!0Rr+81b!z73w3%beh8JCp zugYw3wD!+2?hw%ls>Br1K)h>xk(J{Yz?mraZBr_I1A=OB#oi&gTy9M#+{026Xd={D zGxfba_l{?Pi(}`aYqVB6tHv=}Dq$>J)KR8C4v^+j{~ksV){a=`b&>v7RI{uDUkkIV zs(+**jDml{WD5~(I)7rd+&VaYr?75~G0%MkvPic}x1Hg)+e!e#pwswgHE{+`Elt_y z^;;ufu-%1OR;)GLZPOZ~pXbdOm6LuzqkvW*Ub^D*xd)9rcvt1x^^ZktL)Q&;b$zV) zS&-c$xUz$in&G5rY2ANR-4qAtuh3E-iLDqA^owf^VUBS?NlxnI_rvb8(3V=P<3l}c zw|Zc>@^o2DqMN)GBcX*w@;azx%Gee=K;0v^Z=Jo^7r+wNjHao)>!*}oy zdvR{I2mhY&`o!YpN`w5R@3V-`awiKQ!+hZdPc+p8`4vV#R3KX(5Bq~eex}m#zmln# zAqbzniYY4;{w^+miVdO@OAWzpOHHcd7Fr9M^2GMi0Y|>8W%&ao(2vI%R*SE;qS|F( zV!?L+sqYv!=0md*-w9w7xPmyS^^XTkECWGdLj+`N~K!yU<_l=^s zwEdO@^U{?QMY9D5b#wLy`8XfNn{?yUUcbMPiHdoP%l<561XPac*w2UQjsyCCnSR4` z2cY4e4OKVxs}R6?-bo1(JU_zDL9VV&*+J z3psy)06}O)&?r}t=*kE3X7|z5V)t-SsLPUQ7#T-(&Q`}QUZnt~45!kOCpHlGC`NPU zchJC8>oW`T?yBZ|p~8=f7W{CbumdSy5*=&rv`c=BG(*@6rFF=FlzzdEYD} z5?S;DCbWNOAzg)g+*{A3xXrX!ml-ZzlTnZYDu}CD$7>%sW26-k)E?@k@eB%Kcl9au z9XL8Z6Rn;H_`fvnarZOk7of`gnX`a?!bfOAfCh{r{gM8F7cTU@jTc^RAg`zg){3=` z=OfDW(gBF~NDx|(&t1B~)UtgtSjR*bE^`Z|+$nmOcfkv@B<6a~mHBQh_$nX*r;>qa9YIC1qOl%EwPb0WlNX;cR!*iK9rZ$ke?O@!(I_5B01e@(dl?!n-u#+ zn+40Yz%t)N;?Bwh&m!*bC&#};4&_ii>uk%5II@rs{Q{GLh5C)qWI22~6^uign&Y8R zyG`FVEM*(ITC{;%_AuxhM#;xVLAB%zB!R!?9MZOLUZAt(01^b^P-KrEt4b6Gnt6eA z!kFvmh@@qDBiAid>kTtjQZ7lZ?4btvC@1G+F>GXajQg`79ICu;u!D6?1d=ySB`MIR z{;F_N^3Xq{8lP(*C`e(G84@SuBq}p_e{LpuH+d*4pf|1_0>`JvP=FXdNFEPevzU&u6(LSc1+ z(b#ta$ZBAECgciNri8z;P23Vf;}L9~;J2rYjNj_Vf7gLSCw-1Wr5gihaN(ZbH`mRyjV$ZG(ii$Qz{?rytkzgzG{N2bY)Y>&EMCz19H;0(oR~hF@by2FO zDhi5Q(swky&4&)qXP^=kjDgBQ#&tW%*OIz|ouXylKbP?uUgWKu%aBo9 zDNyQ98qi+)o3vv6kvjKMfvw#oF{$aZ6-?ARfQSV65SVnlZ{>vH$Z;~wsqBLI9uYGI zG;+jn1AJH?#GyyH`OT@s>HXdAByA_rR-f-GofI$k^5dY?ft~z`lNU->0so;w(tMY> z_F1*AjSwl)l3t)QT|ek9bm%Qbk}{aaS)yCk7g4ntL$6udO37^m2s3d(=XBI3yW@)A zljdpi)+E0eOn0M1my4QJ_?du$;=kF^9Eh+V9p45sO*+|2xu1`%7*R|;d{Go6Mr3to z!f-Fv&WvzxFOvfjY=K-ZqImyKVWX*Ig|sVX+BRi7@(8|zxY?bMbL>`L#o{y4-J^u> zrQo`EI4W@1Qh)Mn)MqQ*M_NEz&3P~s2p>A*v$W>%XFEYc9=!MXniwPC_lAslRr#@7 z?q$}%+D&3%@#Ql}v)IQ*F+7WvJ$`)``z)bY$tbO<`89j)t(AMBtD(9O-#I)OzkP`` z{-l7V++0?~LT7R}oB=NcZZ@wuMACC$x1*e9{h|L1)EaeGCL$^;d|$Q|k#1584Y0Do zbD{01!kCh4;2#2Ftyc>|2XCjqFL&EI;n7d5e<(=U#uk-6y-pTMJo-9ncECoaZO98cH+&O~X#&uprJJ}2IJtO_C z9ON>}y&j_a{TNrI{h9NyN*8hiEpZ+1 zxLpXM{8JhuCLiBo+gAl@lF`05>Yh&b5}vu$eL_AjNit)oRm}n@(#tXXh@9=ai7+m@ zICe$)x=4p%lr=;1V`tDY7NP8-K8ezX(`Ch}Su`PhBz%3*s3eO&K%?appMrt@(bg7| zd}aFhkc-ZVO>O`Ss5jsL-sWN1i!Qq(6$q^Pq}7NPrAU-y_K%9 z)_7#B9Qz{$+wwA9&NS19dR?H0SZ#bvm||K1C5~d$l~wAu_ii&0{OXk(RoP&z(yG8g zaa(Be!29&j%T(+C)2_n|b69Ccloy^iWmh*D4Zs5ML!2FWWp7mTTU>bDklW`ljEWgr z+?1%bvgXKqfbJbSC@8t@t*9|sVMXgL*=oVdW&X=DOb|xk*M#*(Ao?sd3>Ue(dqIPh zRQ!h(u`**Dc@TmxJ@&3h>IO|F+KPSS_OR1ydkr_-7PR}|GT=q7g%>TapC%KP`Q$@& zmuCK7B4XhkaIE@^X3IJv*8~=3IGQr111n7m8S~WHav9rtoffIy=bX=5M1LO_zwfI? zF(K92JP%rYnoF79Qm-V$Z~2LN$J|i1c==qL(aW82#kM#U=D=>u|5@%Ot1z3ZxYw>r!C=yGD4Q`TO&^)yRK&I8x7Gx z%CEC*#?^&zQ`{C+8&~>GGIVC-!B4-;-ZH5&Hn`_KwHbb5<$RV=a#sE+i|ecbKKSp- ztBk0xhUvT0x!TmIQF)lQI@w$1%T{lu>Y}aL!f`><0X=tn(ENr?ejhVP=RO>bNOWR= zuzEn$t9# zLP^i)ms6ky&5b6Q3ANk_+o7jTB&z_f(=wC+rtTClSj29ervem<#5AkRbCk6 zfLA5Jz*0B^Oq^r`ShN#Z=c7E3vo_t#m%?-iC z;A+{&)a5LW0{walkNYVIchBtRiw9M7L@T!zw3_M9>(ri8$BMwEj?FmsV*vbZsx0BE-Ng<~D%@C?0Q`Ld2bw*WyV~t?HzAU50KE{ABUi4jm}Ac=RsqNdIQF1!WrNei~=>(Asj! zcXSGl@n9Zw;HVbseBuN`xtaeyw9vx(_ALwzyYBS%Nv)@CSUP^H)Rr(|q0nKED_sQP zJ|NoTClfK)QC3Q7Kz)N@={cK)AT9@N0(vflF&{9YY;0CA3bP_kg(b7ViIk!k5EtV$0jo{g;H0a=Mg9d|p|##ZeZ}yfqUbTeN$2)gRs4|wvsz*Bg_AJd zr+$+UEK;)lyZiS>dGb;O5$4YD$0kqvp?kV4L>rW&Z?$1z^HRCL+y&8>QZn>eGbF6{$FnnsJe zN0S`UYUfdQ*)yuK^$yul0hzWyj=1W{>Rp0{6z$a~w@^+q_u-US^ea;yZsF`A>s zDb2>Egsze_v^Fm6<=G;G$Yv61<(6g!gZ39s*Ygt%hB8qVU6zEiL@`7;=E^ zBi?T|@u8+45x6g=r;o9$M?SfT;yn-Vd? zXW%51+sKVn6L??n5^Y4H?|n>{$yl^t)8>ahUxEU?cPlWb4nqx+9V5*Dt?Ui`;>k23 z4@_2KauM*u9u)$xTk;d_&->PqYAT}kUKlpCFMB}aS}-G@HsNDcSzd8inVC|j^|%zK zr9aNlbfrST#2GeD3*~MsSXz-JwU%!_5!1y%AOxCVTYjz@%Ki8o7;3OG6fD$w+7kkM z!B3GN`r^1aX_&>USzAits7-%rNUacL%cAa#o^8#*)j7cMod(UE0w60y)lRuNk%~k3 z_eFYmT%o+_N-`9wE)h6$Cr3T_MhA^+)n7X?lLvMIH>V-OS>d(QaZkGwasWp%{uHk~ zo{0WbwrKPT|8I_=L5RVckPx0uDuabxvKn0dumrohK`ezevJGw$=2i8>e2;%l)T~E1 z%|8j6XIo5tf8yZ0OSK`FAurtyl9;8=s8nHfP*JWok{y$0(^sn8>T4hZ?57p5c$s?; zT+(D9>P`0g;2TJNVO%L(;kXz@bp5a{ve@}99}Xy`XV{8LcKS9O(X_}E6EXcl57Lhs zw$Fuul=J;I29qSL2{lCQ;Egb7l^}5vMqj(edW{EJO=aL-qua#R$c|kEFih3em)(&w z@bdg!DLWwQ&`Ov%{zMUtqqIg3sf4Xy7P1T%<=Jm$5SQ~gDzCn)?DLkD*zW!qbBt?` z8N*FSI;)_6_524Lopl9^(!ng|HXoypk+8oP0g_}=JMh~4*uM9GyoJSiH3ax?OeUJ) zXzyC1jy#`6^=soAlKX#Z1NUGv;H$0nMDlz37Oj@6pXYH}RD8dh8n(l*GUOxIN~moLd5-Svf*d(RVIdM8yfdHixV zBqnL(v?w)&!{GoS2CBtLg3xS4L-)S0b|i-)MO79vD-01W?kg3u&`6)%nV*b01G~1+ z&Hm*l5aOB=TULjzk+YF4i_=`lYCx63snlsGU8!D~`za=Uh70^o@c>g}qT226- zZt4f9!9C^t-;mj$I)yRQwK@lMUYg!!(;|hi>1)q%rJ_#az~i#wYOK$vWf5I83mS=P zAnN!r9Y@Z%RL$5-)H<^P3GVOIZ2F|{4vi7mLG?=)nprpWCfZ_+rZlE(hZc@?x}#bm z6^{7(9h?Sm9>z=gup(}HtQ0Sm5fQRk2apy#0}^_AIgk6ZPtJ`;M^dsfd=|U;+U9Rc z)q3C#{Cpx6B-kutRMTU)D%ack*BQGL2N*;QoL|4DIz?^X=b}M{GA;#pag9I4kx|Vm zNUULP+MADTv*kn>mFd`wfQ{YpvFwnij$0o6CD|@dP2D4V=nmu#GyZ_)-!m2dT zL9V8*LMiEe9OW^$Cz+j#k^$tsxicWIb9cQ`}4)FgQ8(?F(p7WqtOPjGF@&e3{RK}uA zEe&2gyQYbxFOnrb$Ovw}z4|5hzg(%GW(9}&ox)8#l{i_+hx@I&2&=bG1y^R3^N#@4 zIfml2rZK>!fIS{}gF(>y>+*tNjxAi#pyzQ21k|G0yP1oU1Qf$_-E3)ENZ&wu$g$-N zY$GE-2on0;m>Qd~s?;n9FM!Q5%s0meex5i;*%gWymTtOL+B@%Z{E!~9nv0saIst=g zB(iAQZ-;x3T@u2MKF-q%}L1_bw7{dprJ;L*T9~83e_ zYN+{>j!yWmERzB59Fm>6HLY8>@q2+Lm=D&PZ#3{)IG}wSG46Xi+px}=qQlU9^=O1v ztWhIuQ1y+Y5`79WxKJ%t+>xGC3#-JouHUx#z<7@D5>=;^1$414U|pLPj}(bf{vvzH9qitF zzV*I9a&>R9GH?*Yh^*W`iztLA44)BezSPYAi)-$L;KOa4V!cd^v(V3BUbvw@2}(hf z9cKf(_&0{{?7>gkoM_;kaoF$t^4Lmfz+mJJ>vx86yd}-)Onye^sm;xgudJF^g9}a9 z(wSR?zKWN+K6ivVN#N1h{p_fS6g!vp!_Su8Gh1YojCW~Ex@5*Z5iMIMy+YJgAroT5 zd}AB5;}!s#8)P^ZSB4lUigNAZj_mQ5b^qiGO6McN#nporu{E@p(L@pOAQSce!!RG2z#TBNgjmj;`!%50w=00Z#@!7McxCF0(XJ$rYtCP1La)Hj)iD^f?EdXjp>Jcb21g3Y5OTB84jzk(kqFPsUn`PbP_?-x}y0>c|F0X zMKXP3AvafA9$##u-g5P?j4jrz(ZTUJ(vctTMW{+lP5C zaEK;rY?ZjhRftKN%N$ATh}z;zD8mLPSj!hg4aip+5HvQHsg-ZNzfo) z*J!Ri)uM^n)QkG2C)A|ZyLG(G%Fmgu91cN)7^Xse@`!8sDcxl$ib9V0@!&EYIZ)i0D zCn!taPVd*0#BuPm>3$ckUh!o?l0d5>ky>K-2P%bz+&Gi9P3QpLCN-q>en50-T&49sn>snd^IubaCc_hkXA zbYVzR&XeFXU*Ecuqhv(bg?_ui27p_AIY)&5iw9BI?W?-Y^ zSTfDc@X0jgtp3Im@iYmcKC0l#a{y-GC8(FAuk%RAVhp_yH)iuAXcO+tw37S8PvhlH zf&XB1bF0@J&6R8I?*<~x_fpQ^6M|p7(VC+Sm1YjpbMRRo|Ic$4H|lB6i7M0Um0R(uMrS$C49mz4d%cb&B8Jy|{L4<;6IL>3j1C&$W!jrig z%q$yqL{+}HX!TSr&l;>ISF`vhGcjoKlgH^Pp!&+Ra#hD}&y2+j;`oevJU!QXT{khb?sKaLg~IQQ#sd0<97eKIj`3l)pUh?%$sZq5a(~IJ#lbyR z#$jYe&ab1bVHp+2Sk|uN3ncc)SW>MGQJF*j>vCICm1Y+6GlLBfvF$c;SU140cuJ$0 z$hUGaTlX$Ju~czAm?q`;M4|pzMD{|eZM+DUN^wIc6CMz4A>_+cnqG+owLY{fJfKyN z#?fNmswZW!J7Zob

0hg`=ORUFS&j6}jE2^5Vz**xk_zhH=`S^K82XeNMe|tLJ5c zi}Qmr<^dSdmYD*b6;(wQ0!_*jHHrYW8a#ILe>+9v3{sQuSSo7~F$UpOU{*7LV?+JH z0eNCNC4=8x>~fVbNN4ane=oMW+wlHT0O?N0P~6B($NBHLYmyZ07X2#wdHmQk8N@KD z#i_W^(j?cePftD8XP+xGi902--%73fLUaZ9v zCcns)4w>+gK`0%*Zm{@hj@lEN>q=b8Bb%m+Dw4B=Q zg=x*(8l8%WecJ&68S>8k?68i#ShHYwjF5>>q9jr3pNCt*Pkn;AXpGP>R$EojZ*m(q z(4cdObH2tjs8-Epj-qrgYJ^1;(YW)sQ&r7zj;ZJz$QO$D=4zCW^vQQ$ss#X|E$mF2 z>jZmOJM%K~_sq(xGw-f&0W9QpRhi6jIC6TiS~I>->DTA0n7+TD5x}qj=7zA1r>y7) z!v7{w0Gg#*X-9AKc_bI?lBQ1Z96xy8v(kA3-3V#;Y}tW=DGrJs{Y6MdMaNji=J-=k z@@3H3t~J5T%Fw6mh%A5?sD2qb++|=RP4@P8Bj(V5aLVf;6O_Jj_%$hgeYig&aX~8jibS-)ym%vsCGbRopsg5)Hz8P6_Mejn-&edm;^k8ok)cwyC<`1dKK4WEYt=!%Wb_ z*D>3*Bgd+7*?+;Ri3M!|&EB-alv0A8kPPtteXfRi0*~MxM~ih6wKv~7oBcbmW;%aF z%)t;M**dHI<-n4R|00v*_>2l7oWxlM_E**2mT_0w0 z#!?HlJ$5Q07$*65!pad%Pe&iL&@>qpR!vb~e8X4!ABQB@CoE)$F@hTa9YHLMVQvu` zIdKquNr5*}09 znJxc$u86uhaY9Xlyr*!NqY(daXE^^}y;xSAfhmRg8(vRHKf;LC0=YA~bjNLwQm9w9T%pAqe_$y?#!fiik8W0dg&Q z4*NKv(@=4Db#Xa(Lu9qI9XtgA`_SNB81#Qyos;C?cvMqG zOh&ktYdggM`{fGH;`xAtE73yTprNB^zL$Dmp+Y1$4aw8!HtAp)~+63L&ZeC$|&qK+z-KEz6Xu=d8wZe7Ga-Cdg`Y_CfY8&BMq%*4H+NX+b>joqEq5I zeU394GqsWttQsEz#;fa1NWY08yxvKCf}3Dfd40i&dO~zE3{-#{_Zl0gKPZ)SZ;|W2 z@`*=z-8eD(qkVzjMv1TEd3@~c=rn> z{mo9S7N!_ZO&^b<0}4<|{pN+*HkMO^;!;w=DS4}F8oX0pi;D2weO3fHhyo;P+&Svv zw9F++Of8AV-ffNt{=19a_l?#InWU@?Wh&vYw{LHrt&Vd>($UkgQ%L%AF`GeuaP$!2 z#o%NZezynQ-12iv{9d!GEnILs1;5#oND1lc1)k8Twv=JQ#6gU)73BKcPq^nJULxV)_$<&?(JrK#8oRg~Dz><+`9le0^wMG}OQTJ}J z(R$!Rl^W?pVR5al0L zYCxCrwq59V)2w{uxB1oE+%P%?Z8bMD`)~UasjXN0OMe_wUvbItrVnftk>*asa+>VH z`<{$XY8&?RFx^XFmk=m$T4GsB<5?C6-$=SZSijx*LeG0aL4Ut740+lvmH#WD6Wqv9 zhAO`fap`#*-VRi#Ui{W1;LK!$i+ruC)S*XmbW5-k#Q`nW`VTY6wEWc#;TiZQ+Dv8s zF26LR*BC4U}pc+F)>UhB+dcngGIf-dA=udTFdl=Tu&B>LkE*+2X;WqU|gKZDE=NCidl`8uXqNa zu|k5?q7t6oDWwc>2w=h^)nEd)klDmLukV7+q`0^g<^ij1{ZmEEfGg(4p*~jpYUJh= zR>=fDB_Fd>%|A>Rnj7YxRXCyW{bb}M$!Ay@jMlf-0DPix<6f1!1e)Z9{CM3ioH5eT zldMLp1Yj2m%a*6WlP+E9EmVY=vV7Gnb>0B%sht*DC%MC`&SxLk^8>WG4JJF@QEQDK zHrR!+m^{l=KcB)qxNPai)+@LJkxQ-$!L_x-#h|zw7yJF1Fxez^t=iJ&$}pW~G|JFY zt+ZqqBLa5KfheCfi9X}y%bp0sL$Kh9*@SiOsHH>DqStJ}Eh-7#f7ns^n;#a@2e0gU z^vRFfE$WAOP^O-RxyG}M#$oceYq9ivx#d4g{Sj5v$S||3*;n}i!6It`GICEVV{PX2 zL4;&L1t^8{$=r=cfj}Jo= zw+1?r$*dA9I$~E=VbnolEjw3VK1+E;unXW4G(8b^U6OHGG>_VZk>kEFk*8EIf>QlF zV3T53kw71q(2$G-R{v{S@1zJ-3EOIG60#)Ib_4Mr!L*P>Gs+;6x5sW!W_$@Iz_SwP zbSK3NZ{&r+ub<9*8E7i3PVf+&k4l($>rzVjCN3(p?ZUU2T)%yRa+rc^nzF<2#Wm(4 z%osM=KvP6ZJh!>wg9wXZ@~PLzSyK2?*Ad z)p*^n${A05Xz+Gq12Y!AU^@(|NWS zv{8sJg5Y4d&&>*c4$7yB?+*h{#N&Ong?}|M07XE$zm5S&$CbF^h8eJLU&_uMMw=I; zl$g+ZFF*ZtG%_lPHTQ4Cfp1!mUW?2o~kX3US{sDZDfdZ+j}15)B=^ zn#2`f>nOa1D=%6=1)<55Ec$}(nnM{hen2YheRFM|0W&eg?fvtF0?$t*mi!@5Iu|r) z?fGZO(tas?vU$dCcm|i}o$NsVhd^ zVsOWX1&_5+n1obu(FN1YkB^>~IYtZHVa631hreIAtnlFlp7L$XIhI^8Y6Sl3-T^T2 z?q5_f^H;)7k=cT;X5Jp?-3L0>L)3OWY?l+SiF64g5?N69;SCC)bFIh~w`6Xjw@z|e zBFhONPjx2akbjL=WIM8(*iHU3EF)$_?BGz61e7zt(?8x;xs@i#<$qaVQX4H??8Kme zbQszeDF8iexuqh*1g6Yz*$B`dU$*Ms^s7mJI@=7k$Q0oNF>68Jz6uq;Wo@G*F&8-l-Sjyq>TlMH@qX~75VP^`_QUJHVAkji#&0_X2zztUy-`p&2`eZY0;bf(0Rj+&rA853eFNB-Nm{vodON2Rwn+CA0mdQ`CD{ z(1T*|LQaM*MsnY4omUGRuDclsPz=}~6Q4JXfs#wQ|&Ua`+8O%E?YderUdeIyqujS4>*rZvzt-%RjFwWCNcg_~(G&!wahKSYT z?t^ArrYL;uz(B&_?oP8z*Ymy$GgUu=$5}wMn^E0e?CMesVq^1RYbu}PCa~?*S$?_( z>vbu;?826iiTe(?xjd*%O*N{dF^SHxR%o4>nzY0QGHdqvUuCC25v}@~-Ai?*7V^9> zt)K_;?ovV9O7NDcgep1OMU=`t1U%U4;Jw*~y3WOWU9b>WTB2uoOlq56>MT9d1_R%# zZ<(l8w@;`;l7QM6k4cUG;s|QRX_4DQhF}+DDskoRCwL-mylWf_o=ytclV!>OwKpA_ z5YiE3rCG@BYgT22Yc4$q2BrKQKu8v2>4War>*PfzKm3O3F?&W^_qqzd#X#0CEqGcw zWfS%SWsZ*teW!ML5SRS(juV3R*wC1^EHlO=a!?XH%eghV(fWXAaaw@oICVZZ^iw+Q zM&E$DK@sEXr40k!l<<7nR%QVgu*T9kTu_`-CVm?UXJdGuESwch-pM6y&HboUFRX?BmFNMjpQi z&xh=YyxR2k4+C95gM-aQMJC79x7SwLF^$?M-yHew$C3TuBQF<@x6Exi%j#-1%pb~K zjPcNvB^t^qZ3zTA&xJo@np{O+gwqopkeNUN(3&EkUbJhIc*@w zB+u+J(mNp~&{mkaVG{#R9hsHN2zJF4-ztzp$1Hl%dI-B5HFA9Lrx}LvA9rEnr1Pyv zSmEi0FAc|FAOIu)3Z6)Yi6#0%&72Mk)_7@*;v415N0&=!XT2^Yu+oWTXwV1RS4zqA zQ{=EO23wo^gPsv$CwLQ{MOn#Rz)?DerpG$n_X{GQnf%^KpS_k8kanvoq4 zvdVZL|8;&x{;M6m@KK~qJ{Hao}GbGa~7o_}1$s_aD+%S2*HO!y8I9}2U*?m%&N3qPMlKP|^X<@H4jkY~5h|vtT zzl{avD;0_xr&mkKl-Ji|?DMsw`NBGk-+irla4Yb=e9LInI)0h*(kO1r*~ekIZKjVW zdWb`tXKSGeblv=I+Eh8FM^me^$-!v+y`*ITBG(%;Wkk|awSv=Om4UYsTAM@%;=gB~bL9#;I&ZGryZpp}t_?7m0|# zUDeD?^-BZ|+jaxfkWl#mEDSla519pmW+1r=Yf5QE4K%m1Sl<8=_E_c1W=#J%mkb}G| zhk^--I^BwVb{~%l3av02Nfb26`uSIdF!4YTZ!t%lZmcCQAHeO{RVR#)l5T+*hRrJ5 zptUAlM%2FiS#dS+`ARmgFe@fDQ~6sq$s>MIcVcgC*rXWJD_ix zP%Bp7(^^BJ%#qsG^kJf~Wh3p_e|1m&eP$ zKaiOmdEl=m4_}2)!aY;)AR!&1L{Txtzl>$)@KGpn7jxrNCSqhAL4uHu{QiWVIKfgj z*)Dm7GS|Wlv>G@0V-|Az>vVS#zsm(9-{R<6yj%UX0@%vm3X?`*(s;YAr>`gLK?xi? zNQvy8h`KS4N)~<6u7srd*ZnnB)3pKkb0yT&@nkZ^YCBkE(0~p-t-hCYAzLt@?RXByaq8GRF^{|!=EwgFJM#wm5KjNB=M8E zl++k8#jD2#|E>=gQpSXv9NO;XK+HjQb&sd;h*Wr3c2;nNo3N&saH3l4^=s0aO)QL; zc-U=2iJ<+Sy~U}|JbiCzpaD$+x`{2ml{yX)FnjgRGuzTEwofQKE~AIHE>{j|l0D4k z6)V=RIP3nP2%&hbJktGTRb_sJLp~$W5bZiZJpXJrMvHfG30#P~2D@(1TgwBY`*uPp z=dJkzy>osqAEwzdGx3Hx@4ec;`@0>PE}v~eEmHnI1_ElzUPh?+YB;LuX4#THSlE{% zruTtuiIOUy)BkcL!!FA4m+Dw$DVqF6<|m}>M6vR#{^RdahyGNQ?4 zv_9G43sFo2v4)T2%*h*^=-3J4uLc7Ma@%_J0@8I69t1EhpXc=T-&I@;;DQ9jI&wzE zQ-^MB=Fs!>NK#L_RD3<&Sq}~Wmm;w44w$YlhoR^c9#{YukKGni(dhswOtI41mC;L{6$ z4NHvekyjm$PXX~C4hbI!Ol2-`K0)us#+%^)UC5O;%j1fH)3(|^Mt2fqBu5}M^Sp*)vGoIcEH`f>hDx{$&XjTG8IsxNIv1WF3_ zbbE{I`_#q-fbBIf<9Hl4k66?|^>R*cN~kaNIHdL%l8eKR{!bEa`nbN7TSeQ3F&`n6 zXSZ}FX9$#B%-^{2es7}FP)5=dRx5B_e98kYLz!!@%O8xE;16_H-~KncXMs0jpZzLl zq^620%588FgL#4)CAE%+&7j9DKTeM;25%SqeFOtx#T|O-mD3NzWpKG~HPKcg-$}hk z7$n!J!Q+2d8lANbn0;;r@?=gR>*WQ7Ib80DGsV?rAK#Uj6|z;N5>U0uR`N@Dk22|B^PV^{ zS9HC-jEVXsyYp&DQBkKy$+}MLUDiicb{%_J@`w#}MOmb4-)yB#Ur+}IwzySXFM;d4 zY0{KLXaGZ!N&D$_oP4I5bcosU*Dm;Y>bkG{jZ%eRaR^B0SGFP=`V~(v*`i}mxVTY! zeo(*A5pJ?r>oW(~RUncF9AqBGr^#{A{4{nMouI~fF;gJ>_u&RB#y5tjgX_N5g?R`8 zCYq>jy_tRpJHdiYCto3N=n6fy^^vV*ctlq_Dsb*<pX%S%D9Xx}!cgFDfLmo}j5S%8lhB@@ z!m5LF)YLTBVDDGi3=qwb?%n#53f040BX(ZFokwjNSk}iWTp@)LK~$)QVxV~Kx0EDO zUaK@+d6iubt*Y2>%8}J$ff${IUo4)HE%{~2q0+|XHL}3#Y&u#?8W&QBdc*OfA}Py7 zr3f&UEt|O;F(3jpJ~vw6Ge(0@qCBAlSNnFIAK9}Kz4WUn7WoJ<;C2=&RPQ-+GxAel z(+@_kVA1J<(e@Won5o_+B+R}`(|=Mwz2B(N?8XRv=%wv18&2|j<;~^WKU$)W5(PPV zX6{rD!2#V*AJ@XDTg=q&L$&~DW^fz;)yE-od2mKtB?E`q-T?|slbnMYu+%E=PSK7Q zPcK%>zNlcMy23gfMR{na!YaF+8bq+ROy2<`D|Vew|+IML`bcRxAHd-)sQCnnq3U{=B% zDt>mC@mS9yn~?!ZnhyOBYMLW;l7Z? z>kMp4T&Rfo>-(&VAh7lU4xsq(py&0*<|;J7z9%J|irx%GxoC$o+TaV#==8SFWJhAO zM+s3*+o$!g(~uTRB<@vjlR!1T#t@{=f^U%c8Hf0s($eh`pVI959?lAAc41VY_d!#Z zy!-SP$!*Gfv$$n{jQDm?RYa9OmjoZvhhh?{a;6g)1YZ*`+c&e6&*XP-0qK2n9$Ug! z4!lUSS1Z5qd!!lGGK^#II637FX$4m8ZNaA#HHc;Rqu9fbJoG9cYcFVu--OtuFt8J< z8;kpx#PG|{JGS7J{%+K{PlOIrzROtZ#bJTXU+vz&C-A4Xs{PrL4-3hVdyXYH_QFjM zE!FL(Yl|~W=C4h%{6bg!8+-FlFSW$$uIbBWqxKH3qR$a8t@!t7G$3QRq}*8SiIMW# z{qxlUsq%FX{1xCtq=07}Zq${muu$$Mp8(BfK5C#(bVvl`D!MDURd-kMcUV>VG(}IS zYv?vIW*17s%Sz}rR(e_PMEXtBkIsxj5W+WdUnhWA+8AOPH&`mz;DY^yMyLo~g{lIE z46{5$CyW#kFZ};;%ck=(WpAu>OTfj?)HI&HBg4L#{yuQWkQ8Syk6Kgu_w&O(BqR0M zS&!C@?hwzlsZM)Xk}Dh2{bFZls>+g=H)&|k)It8Q5Oqqw7RuuS{G(-p2Czn@-~z}A zmd;HaiAqnuEAgYa7E*Fm3#sH z&QN?fdbLM=xrb`e|HWmTI< z77*}L@ztw0yFp(SvfCWeFkvNq)wzWv_mxx*%|XRXAG53G%RgK#(IlT3s=6EF1)Q4C zPCthZ0eDzyB=HR%4VO-Bdp*e_#wZ1Y^-leuKwO0)jlq=%Ct>q)KS)_b=~egAKv&z#FK!3 zb88?Vo(Ivg9bHMqVS{=RZT6e6jLq~-jjF($&p{4Lt{8AZ!PS_sR7@O|We=2W9;5>= zXdwJPw$z(&4>Xl>%&RoEg)wu1GsXdxTx(89$Wb2s>(Jipj|gzOz3;*J|HeBau{&M zx@kt#iZp$Q^G8ujoSuA-a>4s0Wcqj?0!B)#5D&6}F_%7dtT2{;T9YZ->Y;P=B>ogn zJTuZsozo`ZKmex46oSdBi04tkhh#vKL`}3&K_irLZPvO%H_(WTw4AvfvWD8M>3AS5 zu1eL|(h%z4!&^viqtU1eQ3Ed#ngJP=7Z2~> zQ$FIwjAJHl&Zd%F6zyI9?{lF&GddgsghQLD4qGRPe^3@}mm%ObRj)t-v`MY1=>pPs zPP_O)Z)nx~!TL`~#08)?F~Xjs`>Tvlb6e!u{g%$^xo59 zyz2ksu(XW9{~ziuD1m%lPR7;c%L0CY6MnqutAYi)s*oaNYT_cq-8F_ep=N`M>vz;O zk?u&EH@2ey?gF51nlb0zkblgvei7~!Uk9H&)klj_GZXJ}F}BifFnw7pGtg{B z17f4F#y(3_+MvRVn2u+Xg!Kiojq$=H`U7dNP1^FMOUI#aU|Xc-Q=pG9CU1a@pzBQ1 zSMmK<92}E4ENqn?6YVon*gW50H^CbB_^yd-5VWH0u)m@W&b7{sbQz{C(Qfna5@wgU z2-NA@(#+uSuO2X?J)!->|9SjEUgoqo+$Bp=rveKVfY8IoJ|qe5GzV`r346kZ^+ucs9^Uk4jCVr)8*m>n2Fw`N;>jPq@+h&DhqoJIju4|B;z_cyS zSyCyJDnU1nplg}6RMZ7Tce1*l_wSQmN-JsgDIi&DM@2f805%~9w%#|@Q~;yU#kWsi z?mC-S(f(Yp7htf_4*NZizZmoLPg8St!TKjR0OL7#WV}3c>lSOB^BCA}S12y5DZ z%L;kT>O=ExoKqVa3>15ZJFpi<$}}M?Pgv$nLBQ#z!Tq8iZD7XW5*TNn9Ip#+kJYxJ$kdhrI-TD*!^8M{aSUAsPaY%Tz>y$(R>G1|`Q&0i(x( z@aMrV{X+}41YIMqmkuc#(&+m)V8m0d28j(ZxEtM|zwC^lz;Uhq$n(aeW5|ev4Z^Zk zN@|UG0FpN?3ZyT%=a32?o!sh-m>==?{X5#AeyU{WdjdTLEZZeGBp0f@%$3yt;bHkU zK1XzRN>u{ifyCBdk3Xj=Ks4yd=&WRW2U8|Rb!mwziVs639Wvp?1isO)MG2ZB;Y?v| zCWt z|2zYk47?0`hlZ;51LBm+vi_v1pD_@Mcptc(y^f5yFd1`Cz?WS0G0K%T@gnT>jg z;grC&EpC9Po@P$-5+bd2T{VV&t}%pfIZ}nj-}{he~aYSLDLvg8S*P z&NLR*dwf};$MF*KZpr|mfTzoNPQgI#OBdrcOmu|l8Hxs|O}rTQY$Z7#IEBCd!4ulZ z%b>>V+W&@P1NF#3@Pl{Uh!Q15Ah*-Mx=2d&X5JUXG7%(FC0+M)$syGcnTAm_X#)+pZ<^IiHgmo+gs zeSP?IZ<$>01{@K9+8UahzP!2-VG;+)j^Xq=ZdLZM1d?HV48IYFeu!2FFxC}^egPHb z?XEYU$&_jjnZ2L>EEBaYF`d3nL4Q{HMgVl$|LJ)}Eyk*}x@-ta(GR_VcrB{P>W z(l{f;^5*oaAA0rnAAlW!pOsC;SO)B9%COtqsjgIWf7%tt5pEEvaW}Y(z=C~uK=@Z! zbwxKPIQ0}q^3|!T23_ZW@r&;ss_Zw5@yHr2s_8e7@>YLY5jU%#g1=)059Qrq$n2UXgz3 zQ<`b)uMsm16%^_q@{RBYmXVo`Ph9x89gNoyyl7<7gj~&*Ol>B_Cv0f^iaS7tll%St zZX1NuH1o+Iw75e2L4o(l>U_)$5xQCUW+z_xaG)#en8l>4?=<1X;fmH5Ucxd)$am1R zt!Sa_N8W;rk&BWQ!r=WJwP4N0HO5cE&PC=@rTiey-4YRj*fLy^C_oFuD1f{;g3^xs zbu#-4NR8I;9K$$!(RIXW??Pgzq=A@=fnIF{RXx&ui64wTl&Y|DXf=}X)arvdP(#(T z6dt>7bNbnPnyn<&AxK}rb3hK&OO3L&8iRix!wPMCvOk~^APU;C@a3}jGl_8bcJIF5 zQ>?)JCNgmz+D?s{<47H8Su2}YsVf*y`n;SRnWxhmEB4TJ=5$DUM7Xv>V%vd!I{5za z>XXQJJyv5&tGuFrBQNGE*XrRl4 z^lGMJ#=oO#Xd(p?}jQT9ym#zf^h!4SLReW)r?~#lJUq{v_{Fyw(OSvT{Lz$`s~6QSiu8 zptmurq)0vyxifELS3+sDqIIEYX(GC#LOm@OUw(poxu5{1B+~-UR%McZ;jXpv$iHg$h_h=oQ1j!a5{2x z2js#BY-Bvn3%C!w&e-VKbeE!BgKsgx;8RU);u#uukHHK7j&5NPuNE<*UOG*2(bfXV zZ>vP=C#1oxb^RTdh)ZLkstrpT?S`j=mF2lOA6yt+CKL4Q{-#=^L@J1HlgL@C_xQ3H zDFUQ+WX8F;T?M9R=ny^<R;%fyVJ-?Fv-&fpCflEQ3lGc2v*CVs8c0s6K1PjEN zM;7S?bTQaSw3UWgApl2chh0m2_Vm>D3F?GxkVshZ&S3p$k*jn@30o<1!z*^%+W`L% zF2&7TtV&cuJ+e0r(1b-e{sQ(B{oYM{ce)htcyiojurR4jJP`~F`kAJ zC518jho#nHQ)o@7!4u~e2H&FGDV6sfg^sgkb>7L4k5?D>xxwiI4Q#QV4ZnD<0pPKl zyV~n^R-fbL*$97I;T+avMjX|<4PW_k=wFZ;hbbS=bT<9=edE8)h7Nx5=_hlqhJB>Df&A^>()@`%+muC5a75MMF%-X{VFQ9c;ut#o(h2xz)K5_ zbl<_Srqrxk9!w1)tp4>B2iZb<{=&A*62(K=8srXFGr1;aZt3YJ=q{xePvk&=3rzC< z+=bCGaDwq>@#0lwPuF>krLE{_zEgQ>GkS~Hv=HVakj)keKQ&!kImPK$k=g0tS&}*9 z1Lwz1I&b3OE0($aagW2@kKHUxcuC-l<-iJc#~@aQcQJb+?mDRB_~8!^MN$gdZh2X? zHln_2bRTWgp>6EAc`POYDKzHfiVR1%V-<(@>!aD3tu>3#Tq*PQGDgq}sAK9{tLh1Y zOeun9tjtQY1qnLS+VV!dCidfRPu*EYwCoAaEx;X1b@HzCQ$9z|(6R1S$Z1xt6F*J~ zg(;tz&_-5}!okj#_a?dh<{x7?({?d@>t77q^36g>kL`g?(W9BvqnnZ&{7V1Sm%NO^ zumsM0usZ76oYkR=3<5K4_j}CPmZ3m{q_|3(f=IW5V(9MO9j1J$5%N+Y+5U_G@YtE? zwZY@11nf+^x1@Agk;D(4uI@xR4MTCCm{Js~N?o_N}4MSFwe z<1DW6gRdKt(g_!~vwjFQ5u*5EIu*Lh}f4h61Z&nf8} zYKPhbK2v!sDj#$aj9Fgkjm42~sDvCT^%9CUhfF?RO)eN&)NXa-Z(mI!uhiZasJO2v zP0+IJByvq=i9CGxrf-ssLz?9uC{po- z=G=$dTo9k$UP#y=ESjFs69QL?xd}Y6gr|*IlfOuJ?Q)%c(6vEVMa8B?h9oOb2gTsp z>DUT4IZp2~*gwu?*aLgp1l`E(S#WkUGZyCE?3B5VQ?6~lXdvdMY_my?D}V2!Ee&Fj$-(+#+r$teW}|01N>8M^F7}1TmkKJupY8c& z?|j=J^hw)&oQPj+lY;xrCjOJeGz-3pt>`nwkBO|dVnm4o5YLWsyc;VYVpa6%*$OB) z!v$53KZc5C$TdzHz;eVtlL3A^IKwRzE9vFKW5fNkE`R-LYETgykHj!b9{zl*`6D%= zdTUNk3&2$vUKF4(4+~~eC>dd2!k9Tz6675SmLabxL3tAI{nXhNU43Mhz2gpJqmQup z3$m}v{8)Tw2fIo59aM2H3T_nZ09|-K=#9gP`RUuOBY2H_4&0{Hk16FP*Cn?q3^$Kc zR#7xTNj3RxeNnm2Ux&`FNSo@3idUx>1()3iYx~PEqnnIrxIBS1%I~agF6>qn8UM?5 zSY<(`GR(Cncg%$^-;?2PJh?G(9R>b<)BD5Ozm(4Zs~3GY;{7MK9WC7O0*B^eshP8H zzSX`R@MxqN=FY7o;Y<{BR|x}ScFBx^{R+M~((jm+tF(@c@NM`QyTmI4Xl??NeUn&D zchQOIvAD)zQE$)OQ_#Fhnk0j035!8wlQSu^vgRF#;>$Z+Pct4t`VM@3?ut^Fsx1v4 zsR@D`apO?~U+jn+C;GT4)6dylPd*rFk&L4i$|%0o?t#-mG&>fYskz(|`;kTML{ws~ zP#S2=S=$9I`1o&#sJEZn-fydBN0CXrIhPi*olNujF656sxWDd-?)CO*Z=w;akZXmr z%86b60pTniFm}LZ-xC>@?H%<0LJ&Qn!9FWrB3)Ox_30ZuUEU2bB^>iaOG z{c;?7y)cb={R{7=;(HrB?+}d@EX|JEcke)tvSqJfS70_Ydz+PZF5I*+{_0(*6tAYf zEU`p8yMn55xIp?U9i#}Dm0oqoaeYo8tN*0zZVCzR|Inp|AvG1G!g*5jsmh=)`Kp`m zmDMk;I?fV6s z-L_b#S7M=)DT>`zlAU0z?dt|GBsTI)EvH`Q>IgJI{}0Udxn^xZ z`&`?rk!LXlqww>bG z%XSpquFGfa@083TEh%BCUBcx4cOw&Mll*mNkDXaAioJ&?1exTsx3atY(@t5z|YD&BTRg|DC5)^G|h> z_!$$V`sf|9ZOJ(Jm8o(F1p#SCF4DDg-Ph5!ne&c=HC<6z>5&OT1TVGQKA6E^?gWlW z*846_Hf2-XYvz{N+ zNOiQ%^KXyD!X`|9plT8pL@~xCu`&Htm<9;{_~X^zQVH};pmtb-|6(4CY3ZK8y}}T& zN%;F1p%l;nmcVDd|UE1mZ0H;I*=2Qz^=Ce$){`-L;U7xuIKojS{*ES$4KeT!n zE?jUee2mxiUm8IZ58h+F9ldYTjQSgTklTE@&=qb)F8G&jfvw0(RAP2tAubUI52ZEB zX1O@g)mHOYmAtabp*x?Fg?72iCy5cf+}&EN?f*+8<}|ef z=WFBC=~+Uz0|}Vd$!PW$cy>g=k~Yrio?eAR2?uNn#kOe0m{+&A< z>W0^y;>a-HeyjbjiOmx!CYmpLd*5@*ODxI-3&cj)pbME=dV8gR@I6W>K*PlzN~c8C_T@+3|IVu3KFe>#B1Kz7geYx;Xe#uv zDuB{()U-d4dnL6Qkx_a(>z`V}_uP~0%?}M8k9PI-#ujXi(l}P!n#)-t)#oF^L{bt| zh@Z3Pyqm`2>*OJlpL|?GOlhM6?SVn0XP_e*FXEfCw(kf4@-~3@vU!XO#d)KKoI<~e6LyHC5Fxlq;te}@ zQScq+G^a<}Brl&yAp#hd5(C3Jc5Nv?HBRN|st@ORNCG=xjh%W9lOqEq9Rge5Kks6#*U!em1^;2V!#I}z^*uBfZt*ei}KK;;X=s4PKw!bh5I~w7o{~9ncxB0LkksE9>fOA93rt*Z!Nv zK@QnQzp#F!%XBhF7(E#NN-=aN2#w^v9nwE)*6V4(!b(Z3oQonWr#t#8xPm>26drlm z-*{ETiZuiQZj@*}pz3XZGQ=O{mw7-kB1=C)iT$GpkWdFmG3>^1P)+*ZPD_$8z&twpes>wX;xANr6PXDwR65I52p zQ9Ljt>@?lW@v6EJQ~f!@RW-F^NSgYVcK2!r2&}Q?5HT^o!`=gm+w$(`>sNjfK~kTE z+?e}k3W9eY0G){H-yWfA4+qN~KA-U_L29D2=)3xX#d2lOnSZKLoz69UfJz|_jwtVsL6{0Y4 zPgQuhd2c9G1WS`M@!u0dJQIC$rBhd|KFnTFLy&TE{ihFNT|Z2Bl+xYjOF++*^DxGg zmOfMUb{9aNH6pgVxCaG^9`W=Jku`{1-+1P5j31(Vg z-dx1`n2=fOtEWmv7CL{e6tyYX!k5gMoh6sIGqZI7V#r*hJh9hn8b6ypGodVvy*z7# zo;d+H+1|Sfvk4Vrmy9~)-RkU!uXh_Q@{;xu_#5v!J+YwvKT4@xY)6Va{o|mwPrv9v z0cxOW*^}&Mn}}RVs%y35wnHGoWv9srCGPg5gkbvLvX@4oIukb7?AP{`^JVsWawMKX zr2Euw56gqPPedM%HI!|}P&ZS0_MYF+%ez#Jr?yzQ4D@)QeKbZMb%Y`PehF}_$YgNCJ+yHttkuUijsdN(f{dZ#Hr_ocfZhncv|RRh$Xk&Lg3jpJkUG%4P-(W#Si4o z1lkWJJ~7fLA86iq=OAOe09G>$=j-m@L)WpuStx`}610re7oaG*&djP1*uTlAt5nfO zg`A*A`GGPOscdENRf(oQKkbvZkfr$b8^o!D#6E|Rw-k9l^cRE8K_pk8mIJPJ%tonr znrkmVHG4G|hjK*~j@xNy8mV^g8NST7S#} zVp5Q@@&R+c(dR+})jKLT>zLnuvh&Lksy($oi{ETni3u16+x4bo%$)4e(`4h649Wj7 zbRN3M%jE5qt2Pl8yFek7K62frpf`L|o1~zS&;m<~zLA}hlUz{j3+FW2N~7{!C|N!0vWHTXqhibQvLBZ;w%Mpi z#m2>AKoV1TA>+mqS-OcSx|CVnF1c{Sh;n-Uqmr|asJclTpFXKPPt{6)pCu? z4GIXfnuetD!+rIR682SG2lt)oOmTHhlLV~3H(L$&5B{n_X2WY{U3r*nAN_Aa)u>6{ z!AV8dhNg+k`oZ0B8vT|b<=RP*Ml3cAYu*@252OiPOihZm;?$vSjMn3sXX(H#hKd#z z(5d_rg{fK|0~tJrjX6S#*oV3rqVTa(__-cB%I?WRcZlPL%YzsiMv5!&!2nka*99^2IHn~(H0B+T${utk4% z1+hHks|&IkY9uADlbQVU)mlGVFrDGkm{761LVO{h+EGJZp%0mq9&g{-)}U+5yf5!y zxRzcGi4Obwtlf)k)V(OlYIrR;9aWc49_n&DzQGlA01CKU!u3vO+FA-oTgv}?uA&WS zSeR?JYn=sB3WAv?>KAxQ(ifjK|Kfc-eYxdUoG0*+d(zy6%uEdQ`Ii0&K;3=2QzdOZ`v*;xYV!ekvv?Gafj zUw0tyGF4e~gY}ae0XTlxB*`%e`f*5>CAi=<5 zlzMn?xdQ~6p*L-w)fnJA&w&a&lR{F;Q!-XI*t|shJ^~fjoLn#MX^>)pcE%RPK*xQp zkr(FSbZj$4tv%klu*-YKt01Spr3(8IG9H%3IG3;EWP>51#P~zmdY27??}`L4{Kq?> zwm2XWhfNTNxp4%uKHKctSq3RmL0~s=b!}Tnwdm9*D2n4qK$YeeUf;wswD?pur zS@fPpdl{w8hxAsl-Pxhc_@9nN^G(>^YC-E^OX-Xsq|r8}K8v};c1r2yxmDYuO@#w( zuC~a0L;9T&1=w+z`I|tU%>{O7V;NWLiTqA)0@sXU)UBqH5H?;5Ly(?F*4^w4+nCzcQEP` z*KDLlly!>5s}yfoYvN-gYo82%hjLUhXV*|*R1Dfgsa&UTB`=Aku*c>+ zvHA3UMbiA$(8N-6AGZ6TZ?$J8_g^>25Us=p6pQ=dF0)=`;c9k|L*lG2KR62JV)TsD z+^`*kyB|{p0BOv1)D~;MGw?}dqw99Ckn7dxFSh2783&@%nsA|C3He(NCMJ}UT;R*p z6DB3vV47}!MJK4OJ#Gz$=B)}1r!!rSoTs4X9ZX3#dfhkM=@jGlAcmdTd~cjmZ(tr7 zN)-fsC7bZ}$VI+DbrmKscBN)K`?|FN3V*i$@zgY;wQw2rNX?ZBx!QFw>Q;@sr`%T5 z-O3M*Rg2Rb=w7tuxh7X-!g-Y?JEi!6z2j1;`opUbYHdd5f>RHoT_`j2*t9dhw%|7% z!4{M}t)7cRlt1xOC`9?J{aEMFf5=cM zL!^O`I@t03w5S14&+mh2Sih@R1-Gx9MH~iW zdsjY%ktDGufj`FYOXCU1ED^`v$5&g)E$~^+v@F({s}M5NwqLqS;*)d!NFrDC#mT&8 ztDULY{*R6k)Q5D3Bpfg87y9r*lZL4RO|?jsa+41# z51#5FYHm>GOK7Xqc%bLYMt0M`MjlHCZ7OKBP=VIoE`i)9m^7EIWM*=l2%pB>UsiVN zA5x~cW_RBn zQeIP-_xAh6J6!BoZH-!CK^+ChtA(L_%w%;7pLZTu*2q&0o&N%{vd7#{YnTodMupAh zx23=$BC(hXqpd)GKdc(7N90Rkg+jfTI~SnE2Q&ktRmih||HmboTTE$PeVDG=3HR6^9Q2Fn5c8eiI z%;Epi%}BBGz4+&G5_A>pX+duLYX&zEkd&NKP<1Q=P&e6nBu+n7+-E4y1 zY;ec>h<4O^%p6@hXlzChy`1YjOtl%eup=10{Yhbs7rECAqKmk)W0wIY_MJHc`uFii{ zF{=--V`!-}*WVrnzY}W_E45j;9@@o=^qWhiwiMQ?C3Ks;L_kN2H}y(KID9}>U{%o7 z#Rty^3JcbydWr@9rBzo8a{gqHOi~{N8it};pjAy~E*RdJgzEj_=CNK)kg%sjZ88pk zbYXNA`_Jn=f{EHGF`w(A3>nDg44tk5@80`L@|}*jv<{78Q+AH_Ojkr%MTO6rcv(1) zpQUxRFDwmMCdo5KL+IQwJ=b&&X}wA7h#*1SiPwKhNu^ex_(KWMZw${AEi^ol*zo5-Xn1cdCGl0utL3fVq5 zE}2INL^@NQ-$tcO(H3l=S>Z7=n>Tmub!r3*E@AO0!oV zuMm}Pz_dU<4dm$GH1Qn?2nXZPS5XZlC`^7!z>#{?p&X!8cxCv(^Cga9Ci$nvM0`Bp z+=Xg+Rz#C2{_2zPA9Rq#dIkqB%0khvKlvlCVfSB0w#YTK-KG9{dgY%+&^wKS5w}5r zz0)%~M3dF`hxdC4idnJcw5zYQ-O14%l%sfD7GofmU4!$mkeH{RSe84it7x7aJJ^=w zp0^Wcs&|dYe1X{slZ9@r=dlY_nmyxEg^FcqR&F~@r)h}qR}ns^y+Qn{9UB$Qcv0im zJs?Ik{K2zxph6nR-Tm!gNjq1iMo22icN)%djXC@xkk(A(;XYRmU?$JUBjy# zi4J{;mVF`k*mQFtkzZ1yj~&|9>Wrgk3IU_DB2l8A>I5&(I*+e~aW%36hnm1zem;cw zR|UWHRHfvFb{SI*Bz$hd&=C#(60|UkPbe7LinUAqrG<}wKykXE2vZ{bD8HpPM|}*J3NF38K6D4U&87l zJWm1zZ+^0s9#=vA6J9f=`gzU^6{4+6Ny)~5q*HmYd#;m?(rv`T{;#@fMSR_Kkqruj zZx;7yOPAT={g%`fM`(7I>7^X>{I(C z%>&6h5W>pQQm#W0hb;}bm1eL$SA@cyN5eN<)E@E~2jBfsgXO=)VqQMlYfwx1exi~w zcL9<}y7X1avC|_%l82aZqpq9C7w2b`$|Q-%cJcnq-+BFUQBD2?yFj+)hPS9Xx1f6NE~f3x=WeB7;!;QS_aYc%lz81HsQ>(X$Q}T9!sL|w3FZ}ILTr2)T5OOBro~D z0Jt&S9*QP`mk%AEphJ53!SBSBLvG16d*@ZOJddO*9NcNapfZPx1WRTK<@o2>WyVel zJ;ai3)qvf1$(m4#b9gn&TWzR##Jr!>b@TSL{H*`p5eWH#YOSjF?I+04#=y1wQ31Zm zop-4$-R0N*WTy3_%>8s?$!cc;u>51rACnS**&3=NE=A1CsOOUlfJ1sRF^m`5>zaHV zoE;&)Ijhq>y1(=Plqlg)rWnMCihhZ@@~IpJh?!vl&q(VvmpUKo9>q}LK>8NRr7Boy zgYar9lptn5-_g#q{}cO%pKfxk3AdgA(~v+8$o#ulXSmzd-xQA8C0i249uF4)-ReNh zgV@oIoUri?)enGDf}cJRJ7SMe>9@dD?_NpX^kf^XCU{T_b-b$dXoKkXqC@gr1cCJF zJPzp}pK8}Drenu6!i`1K|8!=c#KXCKzJ%0?`u}<{8bkb09vpPPxYrgl$qj0~F(Y8Y z$j1}U9TsWMPx;QuYCGms4Olclqxwlvs$O^>Gzg>ls+Z73D5>;t*GVYbvY0HeeCo=$Bo zCYxnU)!}aZ7-|qMLYrgY?3ciVHyEZWXE}rQeJD6<4;Eedz12yiS#0o>OK)t7_Sn@N zqi>KN)@L8}_m|l!ES1GUF&#Ey^zbLjxX!-ku{3qS$zU|^_8S78^>%j)BFRw>v5<9I z9m9A@9qQqebfx8&0{K5=t@ZNU9Oy~x47q7_!o%(`dly<2+~qPE9(+N%YFhE8O7Pyg zpQyvw*|;B3KjmFaid4>hEo6P9nsRrKf6bg{uf7Jl`}-+@t4K zS1S8sV`5x@A?O$Z@OIIc8C#n6OkXevJwy(o3>sKVuzuG46u$+EJ2|SY&|z)l1X<}X zbL4ieJ*T}f+7CS=#YqY+7Xbz{2r+c+D0}w*3g%;HycuO7xZw7vDM8aEB_v~s)Lf2} ze95Kj{=X0BD9moN)_2%EENzOQ;^*h>HmwqAt0qnG{)%&oUwCzF$P)bCO1mEz)t|l% zd3aWdi%V!ceS%qPXchP#IRTWN$dHgO-RNCXV4@MtO|L(7^EZ7+sSasfs-eyPi6VW3 zGZoSMl~O;Ir$jbHFjm|OHSnLOQkv&qe{UtNbhCyG(0Aa2Ta2%{IMYZ37n7_rvTT*m zS3Y=xY^{Iv6yyRHxDe!qbjiOf(>a1r5~Cw^OH{rK~_Rl}S<& zCO}TQ;cHI4QA5~|zkArOLkA`xtsQ|BT4dv`vGT#9X{ljk>fofS^BTf zet>&+0)RLZ6zPvkz)Ee@Q?eknMIlrCd4&k6j5J5IJeRFjpl)xh&>hIW+J7oy;W?|K5iZ3$y4-O31sKc?xY-cV(e#8 zPC0L_j^n>Y=k+p~nPy=MbFm74LsY}w1yIa!HAvu>K*H4GPDAS;0gpLK-uU6mlbyQ_9X*0Qb=C;A1~bF9W~NFx~mG?t^Z@gM-v(0WxsZ<{J_1R zu+(r|!Ey??Dqq2r6U1v4hrLV-+P?LP?3l89wr7~aN!LqPawUy)+n>aA{kv=}od1zc z6TWnvkj@x;&EHfeAJpBnIzRu@;VhnGyM<$77_^*#x)s=tP9n4@U~0FVxA#9OCK9&2 zK^{QednILg1$WpZcXus|MmOKTVcAMApiwZ!I(EDALF%e+2Kg`fBofso3nb8JZepqL zyB?yqnv;g&tpVTby)EU)jli+Z=@y}(ALHbf^NK|9O6s9c{eHYpof#~W?o=Xy-=)`G2Y+9|*BF0@rrIMbC41cr-C;o>!E zsD4plMF6$isd!-?`Yl0qQyC;$w|un7Xw1+=6H^@rBTP3XNV!98^zl9;jz^0^YsQ@vp?p8-3)i>WyaWE)rLT18>kt=H$|m<>&HJ)!>pF^21QN47X_` z39m7h-2ZhYsLPcS;3-2NIV-SMwzQ;d;$>+Y(lAihIHE5Wem5gC(_fNbFY+bv$6w2b zuU>@&o6A4W-!jA;Xm7d!jac-eX+>1fC4)E3 zVG7h>{q=Qk3h<_gjz8+LJ?7$>IAD~*^p)Rc^8Q1K>IV{ebz zV46hPrwa})tne9s&_<%^l%f7Ktdx_$XS+)D5bY%5SxfB^4)WAdsw#95UiC}tIf z6z36aijSnPTqw?eV=x{dikW5pz^#^>rdqm?sSZ>j!9X8-rln&? zNlv7{VoGgY#jw1o-$~p!-BeKxN#7T`=6orhnE$kb@SarP9qJtNk_+Hbo{VT?%C&V_g?=(Hbd|Lh7ARGw$i;`FbvZG7qgOdYSJ7l-cjbLwZy-w4B{@-&g- z^;ieP8PIOoViGl1CX`j~80%wXCsyN}!i8(ZzPzoO=asSD7en(P0@*(mrFWbuM)EEn zW!N#jG)I6-agzNc!%@jW^ik;*09Pf(z1nzeJT)^SOyPb5tVN^ob{J7b2M&^{H-p7x zjfSJnW;xJLwYh-QS3HQ~=PYe%nO~t|*YTEwPJ) zStpMzD)qFgN=>p6uu>k3HlQIbL)PsO`Ma56(g5+ZHYV$Uz@qm~mbX$h1*Q zeMMtcbqS^*yX#7B4G>pf;mSmx;#rmScuQh zmKMIRchPv-3}aKD#7~-9t^@lz7f%PR4S_;EEHRMmDVjsp8nD)mMd(7!Mnd6Opv@ZJ zP>+Ll#z{Sdt^QlqKufiTVn1m|Ibx~=*InBKVVEt#6nD}M-Gu`V=Wl{{G0 z70lUmNeT&JQUo`o)_#F6HTol|EROV}l3u!{;yV?H`wdJAevWYa2ooul76K&UD(imT zAJboo0!5QIt0>G(Jz0E3 z89qR>)18&8iTbIcCnx!LpXS(WZxN9W?^bM;H6S4@xyWbTWrdxzDPxJ&FDi`a{6ZmslnFWJ7}OBZg#M4 z9yLVZ*ZC&!VU2C9(gHKri34@7_OTG>Ki7_YWBg%D)LiyeO_^aWie2b}^?`Q=CL3D-=Pu%)*=PR=5%_aQsMYCXKVJC7J$|KY$}I@ z)4Q<)^RR(Xu0PKw@518VCdi0r2&ye^`D8*QVC-IFoQcl})ehMbPk(x=Cmy$JnsL*5 z=qCqRA4V!j8ZO&<;3^5>CrJi~Xz>mWMyG|FuT_l$#GMDz{K;hpU44GQX`C+#6l2k# zN}#+fG>oTCiWbSvmhz`zc#QkPa9;GCVW~5*O0cE11zb#&ppBlvhsV+I+NmwHY~5WV zhJiI;!`}(gcAs(Sd|rcv-q2s0*>ZAelun9XA@ED%by5LZ6a(;rsNstwocP-1wl;NZ zG`bpS*m}IxbiJgFHIb(-2Q3hQd*U1v38k5(u$N6znxN4aA-MaBt9m;Zr~jQ1wsUBCE0_I}cNql4vzK zxeddifAG!pz#No9549(0S=dHIZx>WZO}WjND?F@g1iakE+u@@gijqvVv~tskSI-bC z2eo1H08fWh1Ryr=?^yEHuCE^|W*>YjbzMEEynL5&d9p`>yiG#~tAB+!xVHs_`@dOD z3$EW)pW(fxhwd;IWbtCLHBTsqiQwnhL?YjAm!Ra)3DIAeXQ)$`JL)TrE0F9cCz-sC@gLgyauQzU^a%T&oe&EGxfCy{h|%eP6_4KPp}s$R5Qj`ZI;a>oQG_$aTH9XbA3vEcqqhRhU%y6gQ1wJXE#S+FG6p? zvsrDp@NMrSfaQN-nFQN0Qvq`cuIQiI&DSX$)&7VTIy&pfqz;HLJrJaC!;fOF#>iK~ zc@bEzFn6g5Uu;+=a&IQ&6Ny9f3J9fjXCs5{i)G0E>G`8#9CbwU{AF(dulVFd{|0!m zRy}Kg>8iHMCA#-e5V|$UrIwRa^iaTdI)GI`ZV44%`bOA%;buy9>B^te9rVwVd#`2y zrtKD2_D5cN$`WfNV&FaTFUoF4*Z7?O%`?V2+UzygIiV$Ds;vO*W_UpR>u)zhcnb?PatX75ne&0r=#08{!k5jZH=F79oQS>g(@7Wnz{8 zF}X+<82~Y#r)h73C;{OH+9s=j5aj z+pJ!fL3L4$Y7x>`U&Vgb0W96Aqx?y7Pil@X^^WvEN->4C>a0x+bj%5$5|M9Sj*QZcXq$^t`kCHi3~gCa}MAjA~b=Q;z@s%l`nUK8{$`l zFe=1gs3e`G6b(#86S#KWGC=ejlIYYAw+HG9E{6Zr=}v+%u)a?K3U;lE)FCU?bg#eW zx(5=k|2G^t^l4{HN!FuT$!)tCQG({c+))t|Ag?e`6AtJTim3q}C8+vK_%d(gj7QaB zMR_hQhp=k!DjfYj^v}wbIcomnnX+!KXWb)c=cSf-aZc}S#~CQZuPmzEJVvEQ6D`5f}h- zS?>$l`AK5`=n&$NGJt>P&_Q2U?mf)cV+fdxaA7*Xpjh9q{LWuyQ^_iT&;aG9%*k)b zA4msFg<%a)#LB*y;+wL{)K+Q>e~+dj`bxBO+WI9?0A~+xl!jWztO3M>x%DDMJoPAc zz-`JEYe(ga(6#_MznTXPV$^V*k%kK@65!MPNLwr~k1 zJ*Z@y%$71^tKPk6h_b_v9`&3`nI1&5dXY&rL`GTm`DWD4CgNZ87$1yY>~VDvHlnt( zLXF}l$hHbcMYB*p?f0|4{d!NH4mW&A2E8@iRoea&DpaUf*xANX!=6Jm|4C?%l3t!y zb~R#~L&s9WAyJdGA< zXuvBg&u9Igr75hj^cis!@hY(8ZF(yq?<8&3;3&*~SA|Hk!zv(HJf9otN-P^-Bs**& zEpKG{8?c654TpN`VC1}^linDV0v9=LHefGo@*p8cE^eCWJ73)IL4}#$dyO+}@Bo*d zN9WFsQGs@c%u3#1ThT?r(%HCPde?{4wJeAZ^eFv* z25f{T_ib*lpiOW}BJ}c|NG_A%fIBB@H3&bg2PG>a^tG7 z^ViGM05#bM75OzadG;7B=p#2S1mqFrsg?{~g;I?jKrehQnk`hg-MYnQ{hVd;-`n0f zL3>!#obBE?HFb0UN_uTta%Pgo@Q(R?=G~L&j@iHrxxDPPKB|``+;tsBdeR;#p0UWW zcL_S+ex7%}W$r8}Vun0SH9e7C&)W3emqR+Ka8{Qf!CX;g%Sax{DS6^x{tdBg_}ZD% z1S>W{t&Aq)hfkFCgtCbn0;;p$k3QKiPU?Usw}QNSj6lgpJep(a$fs zl+`G{+HYBa-%5spbfYi!kud$tfQ^;5QlqGew&CjCW+}fx%2+puStuI0EeisV+*4%^oCHa$fDRdXw9oNu(TO#dG*wn{j-Nfm zT4-1^@2UG3c9m8FHJGbx5sa{BfW=&=NfzdI8x-k8;Sp+ZoU`J{Ai%dmqx1C%B~y~~ zi3sr{K-COzM)cl^*%w^LUpgsg=PfOIm3!XGm2St*A`+HfiURf-5IwJ8sh^~%?&20K z-%9EWpI%PByYOUV!on3xgXB@k|lx z+)2DQd}P!e$n$E+6BT?YRcd&OWlnHy#{|gjD_8nIwTNj;Mhq}T>yoa2rq<|7P(zkWqC%&$Y!C=2rPw$PQLc zpv%d&XWh{|jCXhKDs`Y_p&e=E(w$ChPlOr?}~EN$oNPYb=h4zR8H z|NW?PU(ON5du447Qfje0g4rJdaG0QIUL7*zQpQMYn8ucsk-SMco3oC70zMZfIE)|y z$0j63jh9Z+rN>5Wg~?IqtcTmob7a9pL~xt-fso=WKoTir;d)TRjE%I8 z4(pF+uWgVQZt=Z;6tA zAlx%#B#>RH;JuGF1c(#Q+fcfNsY|ge`IKmbs3jsr!CYiuRHa89xc2*guxH(-8?EN= z6N;P^E=6>e(}}`}ybr)=h4W{i1Y`VH9+D)}pA6~6-+K6s{GM^gU{~`2U{Q9m;WQF1 z8V|;TF5#+2InkpLygpDEL}qC4fFY=>VCg_qF*pFtX2d&w{og6d)Cbz>$6CoPJ@drL z>mkOI=lebPOzNynuG*hK)x;z`xWlr{V4#;oh-(28xrN}+T$<@m!H$&#mmr`xUQU1& zeq!}R&nzc9Ic}Ss_$zCE82Pc{cG2cS*QZ5GMuAN44 z!i4Szj&AcLNGv7wZ{Gz;P6Yy|Wb(d5krkWJv;kxUPORbNWH)wcg!w~1V@R6MY9(YB z!;?i-(-ZAvze1nF!Qrn?;1je2Y9014396x*NuXwKJj9!Ov1BtEK_4KRxHH9v_q4z+ z_!9Rzi=cNHI+Hf)vs4)Y1=`>XC+4^05br~RyLok5ks z!~S;Nji{>760dQ~ac(sijR85Kje}VeZno=0WLcv%(o-HN?A@$CUA%;UC*&*<9q%l{ zMatH=6>nM&%tL($=@K2~pDM&)TR9u%{w44QxjV93@K#*uZ7v&8ijXGd=?l1_>k*}H zlSp+~`f%7&4CRtA;iXqu!l|i02kcz8CKfJ?v?&MFP-jFPv6&G1 z=8kov+Vm!;hy)RA8KfS2Q+(LCJ`mOdOrnU;0tWfZ z|J|+gVHCe3zu(C)HU0HHeV?&B0cdzewW}>Al(Zexo98-V+xrA0@r=6Y7Wcs}(HR6F zDzBVW@|4Kx=|2e*>qv&n(u`aUHQt|I6a#%u+KhQ>94SpE#SD5RqZOyA-XRoUzJq}L z#*P??GGOD}t0>939N*K~;3)P;0iN!KTV2)5miCc+mCSE4f>st+RXI?Jn=Ex+Py8$o zJqY4;9KBY4o`vF6W5Iy8JY;ZGP6$B&DNf$fJ6-uJ!NJ~(p=RRXSC))SqZn9r?+z($ z`M$(g_|?P}E!=x&A)aWS&pDJFGI2n{GOA!|aFuTArw9U4WaNQe1_0 zed)LB8IT-zved!tuRd*J4&duprJFh|8#oaE^}{5RPY`piRAfSshuU;M?Nq9Fa~jygnIYzTu7!B%hQwioLU3FZuOE*s>N38bJ}x2N zR~6hShz9KaB5U;U1QO?0bT2_knA=q}0XpP6FYoNFE`ZiESQ8(r5ls}CAEvA-q`{C~ z8UMzOQLE}N*}H;tw$HJ5+bHR6ksk4I785Sn^1p$#QyJZ>*OMUkWBR|#4!(?%gLF>| zQoPcBYI18I1bS1^Df`5b?V5Zi`soh1lA33c_!MJ)lEGbg4>Bi1m{PQ&LJnC6Xl@mK~FDQ&5{>U5fa!hdg2jyeje5y^$l#tKpT! zIY+ax_Plkk)u3}Nu)mJshqw7|yv-@o)t$jvGr6b#la^~V;kgS*B=#FTIwQ3B)Z}Ff{EQ)~u0Op)NLUi{H%fFi`fQy6Z zi0_c;8xkB_6F=lW)6ANV=gA8I;KGj8a!vtFB=x!2C2hiE2_!E|&d#P>pUZb_|q+k&qYH~G-wxSqQf z3g60R$eW3)W(mibYmtQ;CnQoO281IvoJyR+4HS%yY6~uK_8n`uD|ElbexWlYn$-<5 z*)r;E{OB$MiB|o?Wn$4{?u;w)=(Ry{4Xz-441&n*3>3(j;G-s|C=t0`p6AZ z_m{Oh^Zpl&1-w|8VJ~D$fRFc*B0g*g%mifrdrlB=LwO4zAPf?`@A1}mQ>phHyu8s) zI?TqZ=vm4uE=nfGHWzreTPJ1KU?{WG(Ic_YkhfJqgH5u4LURoEL<^E=c0}+eps=#M zflU=p15=K`eGDCQuLXnV64rk6ix4p$%oLprM<=?X zk(S+ELO1?=Prs5gT!0y#q4mz?c73zL*1r_JQZz$%luI9#?<@E*X$beD!4Kx+P(FeUR+EQnpx7vgja zZh>OnAL+yraxy(*;pyN$z3@&+{x=JZ)z~hINAB3Zu=YMeOK|Y8=sfk%U+l;e1PgHE zI;x>H5E8iI4wFzSV*hMcd+|e0YfYbc26H(A znkW!gUqCQ@aC0Ki$cd#4`tWjCT5cb370e*FsV=L*dXm1qZ_Cw31fkb&TH&Wk1F@w3 z+YjF2P}|N&00u7+0X8fa8~{(ZpsUSx$Bgx{$hyn?b)&{9WN2hjQB`@jbN`Ze)~j`p zmNKy&71=`9U~DXe^U}jn!Whdje6jh$k07;2r>EGoGdi<@Y+~h#8kJ<>P#Ak*w#Yu# zFT_m}EKiKD7-$DscONo@w&-|_S^3|@n}|zWl6b@T2vH27-R&YYT7M<@I+DmOjg&xj zgnIm(rQ4~)tU-4}|Ai=JazP=60!GV0vX?8IG+|ZlGc9vRZjLcWNhl{J`uH#P5-MXa z9BxoH4}VoNvAOImDwV#e5O|Wso)U;My&NvX-vjbWz;*CRJ0pce)WNGTPD=kw_Ix!U zBs+R;C@Qu1A_i(RCfIOV16atWQhLRA?+VOkb+(528bhi78N{`(9tzDJ$q8~D9CeU=j zgr4e$rbp!r75dh>3%T7efIsQTvT6nu&g7Wmm7148AO9p5np#_UvzONOrhIX`0h31T z#*vc=1M%Hkv@d7zswc^Ql*0)#*R<9L*J}%z*QZ3%ZxjXJ(&194?wQjuLjC$u4Z>aB znW8x~!gsEfi{SavY_rea<|g|NsM|E4c3hb-kkQGC!gDuyw8KexQu1#zafF)L@lQu? zaoyIOjDY`quc=iMF5Xl*UI=x3XvP9xBx4D0bRU@b*0TNHr`k)s|B$4EP zklkQ6%bh~!FeoVcIdFsuEIPe(__?zBbtfQm~FwN z@oI0v;~_A9HNzbHM0aw8GH|eMuL@L1`U)~*$WdYGX^l5u|CFPLV{)EmaTmL-90v{$ zX_?#B3%U0Gp?fdBx~OM%9)-IT;g=_RU12+#F?}bO3vIYo2HJ7je;K@$a|vzBrqf~> z2j5dW%{399j`ZwM9kW~PJ1L`Rd-^MMP4HSm%zr-UE1d!ckvHt$wfQqcN|pr;J}z9} zkL%)TldzE4ey<`4tqVVW4re8kHHJH?vhFZ~mF)4B_9i-x>*H>&oNLH(GFtb1wXANz zAyCy9Kxk@(TfSV5(=JiPAp|TP*$>XRZve*jt;8Dk^^9t(xooEa<0rn{Ij4z1QL##5 zS%C-M(-JEUQ@kzEY&?4%U!Opz{Vhv{awcI-_tp6&?A@TsmVzJvWPme*f}Lt8EeRG- z8BZQl4jEgAj`$%aU0^5Pul$$#xW^tg)r4+xGlNz)vc%)MO^)cXS%Kg4#Dyt1LAA` z%?A;Q{#v++nx7ihIt)h!O^Bom0AbK3tAMW-3OGuGYAqjqRZv`~S;7F|chBVUch-j% zrQrLn82VMJh{i}2B&^h+1YY}eUY_%rwn>}Y7sAL8ttfcD$&btToq)fGFxSj=6Jf5x zo)!m>ei=;M$A9!keaM-0Lq?(3K>fwk+*mu?)-2|L>k2`aASxB7B8A$}D)Qw$ zwtQNOZ4gtKL{*PuPS3K@sJaU(62x+J;N8Oocck?aEi^0Z{?)w4sEyl6s*u)T0>-g6 z_}Su|=AgrsAWp0ij~@hOW5>tmJp`DD^^WNJS)ln;UD}tP7U`QcD$m60w>G)Fo2CXjxkxNPVmg|LBo3m@ z@C{xy+1M(@T>&qz1!@L)6{H#~r)>Ed{ZiJJ%MbM%piyHK`U2JH%$gmT-Kty&JUV@seO zm=8z!X6!{f8Pq2d-1Dy)L>47+U^lS&eUZ6TdB`=eQZD8Ftw8niStZ_;?AZJlefBLZ zwY^?OFt2voZ_BemN~!JV?05fHY?FeR28y-_7j$i|#)H;}@uy9jkHwTRtO;a~7|X=T zB+<4?Am zu}>S82}cte{mdiN@fu`{t;X*wCb4O0T>|s|?3`ID2+BxMNrrPxNdg7u%x=h#MeigV z6B{Kuz*7(B3n-AK^VD1?oloi@5W{|)CA^kO!59ncBsG9cY42g_2{3X5lOqkIHD?&> zOH6Z0t!Ri=m%U^>*lYXTy^df#BM9^QqKfdUaqqu&ayF0^NZdYsn@5*!Q9v?rSed;- zZu>pt#%O7Ig?~fYl6P_0cqFI(_CA4D7XmF*TsH8&o&|o{0%8^Rs5-MP$HOV9rwkyZ ze0hAYZMZmy$J~RnYTOIE5mSrx^+eP(K^39_wNBz*MRRRV#a5?^t)6wyp&-EaVPq}Q znGM6BlZfH%gQFq#0dA$C4Ui4F>{`F~&i!x%y#smEao7YzpYb_x4=&XPW$>YURI?@F(V9%7?76_Y}F0z`^w&f_*O=L!a3_#j_e3A&HoY6fT zFL_X82Rx*sqM)db2Pg)ypHQAffK^CLuv-fzYQVh^acjdmBQ;9W3mtZ+v&{419B@Nj?;A`{{S zC4-)FdEc;D0?VwoY^EC=w?xzP5QU6R*oBMm#yQ^%<=-{zDk6eLP>l6AW-5^-AVyUI zj>Yfb-tfhKEDBDJd%Vxt%cVQn1@2gjrqo~e8N1)j4|xf{$M%6g+3_0;EP`RBYYt!o zA~?HM_rrUmuF7y+S9g<8oWZ3kEG{xV)O@4%4zJS_@nnwb=2NCF0Dc`1bduCpYkLsU zE6_a7FeJ}%dh+sGDhMC}R)l5}eM_ZPZF`JYDIft!vv87+A#eYFD(fh~OV){xi;KPB zqY>udH8%fpDy+3slhKo)zu#emqR-QI>;ho9UN`oSmX2U1)rW_={*Sa5stf`oo7TIZ zeEW!j##o-&BMOc?3HZFq!z$NJuS&dfS!1j)`m$2snPaG4xZ+BT)Zh)16?EkH*utAm zB%x$-N2frpHww*x;(ELa4P3fi!AM)CjrIKny(idwr?(rMJ4v{n??pCYKSF>nb#RB3 zk*JdM3cxNt4)lg5As!I@jLQ(jAhIlDAN+AhT9ooqcI#eC25Wwjz{y$RA;?qn^slJN zV`fCr5P8M>U6y4#zQ+L*OcZnuSPu=OceKcaM2A9xu?lf!hxUX4Wby*YpT7-Hg#G`w z?IQopE1GEJ6P^Yj;XKKbIoJCPYqo+zdIH~ubXbA^_Cylb3rR2d>C+3@fc4=l2dUeK z^Ydj-vcVH3=xrWrEo+dS@%GWkATKwIH;C_dIV{(?o?r@T80ngQ`nkWDn^pPi#uAKE z=$Q=Ni*ph|*@?b;BV>7>jg#kKs7v>_PV68v*phl|501gGW**}D$6;8s$NbGETYUuq z`OG!&urQTT<}ELET+;(h`EFceQzT6aJ1MiMxz?5?04r~WT8uPkm8NL0x6_3f1x<0M zvEcYnoGeN(SO;#2S&?a%fm#{IO9RXvb#kAS|NphemQ#JZ5%7t(n!3s$tbmD*U04_P zvI_2@8S)VlA^;+s3BR#DmJ2H@RFV+GttPqDi8IOKm^$s=M5}lME$$egN|)IGT4aak z9di5s6eThOWIIp z)a3VkTF5%U=Kifgvi1fz$M0QsCFYP-Cn2Uk1`N83rwOLzMy{4e60( zfAzF$oHs6`a6o$NaG}8{J?-cUKk7p7`dgw!!ijy}AZh-(iTk|;#W?V zdnhF<_hK1~^U;7dT2-jIK`3C~BC!2PqX$r6fY zj{))P!?CA=qq~3NLBOBx&SRe(!Z4NGr-?Jup635?@329&ym}pcPMhX zDnw2|wzX(@%@7mMg$@0y3wB-fJ0+=Fx6_zi-6KLmCbqqSYG+1W7E(#%+4}vERs}Xa zIWx{f=R@RimFG&bX4=HU^#+@dQVc6$M*uHC(7!?JJxuOrp@fZ?@`BCy)&8Nq2O2?m zz?Op0NU~y&D843LDc%m!s-})OUDgOg6~DC48Cn@u>fdw%Gr52KBY?I!jXW$ldwReS z8Oa_--ZXyCe`zt<`k~4z1gBsS*VUKDSnumTI70Pe=wZQI0y1i(Pjo8-z*ZGMaykjq zf;3&vP{0wzcv4;;YdfL+sB`NV1c(=orZK*xVujnDo~1jI|G%ctzl&TYUkT%2r#krk z1Y^9DV@QJbN*=bNUE89xar$6+A7kbrJ7?wtj!t(H7hY7>`%2Nz5A7jgnWZgzI#;5} zzfPg`et)w;M^H?V+(25hzSQ$lQJq60sflrqeWUOwXX#Bse5bs$sM9JkkDvr{>MOxn z;&VK}DOQey@+(i2e_hJ|G#|;7XPY{?35~>2$%7JSFiA_Ul}jO&6*~s0!xEg@j;a|3 z_uKE!UNH}9JHwZRin36LKKAzxyVy-Y%CggxW zBn=tinHa1y@Lx*cXPHNVFe2I=G4FZDfFr`XObDapw;t;0>R@?=R`{DF5C)Y{Yh8?m zVp73#qb6)3pM}KNZCh7LY?@l}4r{ePvd8KrZfCmQM)K8F4^}q_pn<&|s|YC$owLod78Q?!sKJG}HG<8p!T=bjagZo5QYbTH!IRG=%@@sjZ%bd2NqSP9aLL;n# zf=iOmUfP)kcg&h6N!JiV2LPIEALwak~1_jT8ELSVEE_42i^Jh67`Ak3`tF^Qt&GJx)y~+vCqG9I&7OT|=-@ zjjNT~w%1*?*Jg`&^<;CuikD07W$!?}6(L}6lf)A@qo=#MzPD$N_dPWKUnwoanBrlW zWks$&&Q|;Helz?xBq&6mI^3SxG>lEEIY)Pf%zNy781-{aj^lfZ6~k#n&L%>Ka57{E z_i1icR=Nmw7>;X`#X=?^gkz>$R%+s#GBu+pJbqyh=--Ufls^(#TJWZ8?kgofY~ zTw6L24#SE&5yRKQ(h8!$<(@WsVfuP0!27ACp$=x5rvrlF9~uVTT};-0%i4=ljY2@c zi9TYq8(=;OYYOX&^D@!Ne~>g+uaMMPw5e+ncOP?|m_hnjVD zaSPpjbQ^ju&aTW3^9f)oa`P^o)D4w}<4e9KL{vqW`aEfdfGr^FI1NA(2Ok5Dd*#*)RrXpGN%99pe3k@9PeYS$LjD zK*Z(vs%$^aDS7XW`zLAas5ocG};e`QfMWONkx#UdZA5@}s90?L`jF z^KI>b{zegl@{VOgU4T<=tNZ;t##t(0t)cRdX)+ko#!k6Q?LisuZhks=B9y8Dc5q}| zM8U0pQ~DYt)byMPw6R$HqaGTbxQ1%zZp+o(8Ys>gL?* zNuilTxjcfo;)i?S_PYN1aFVbyt~=VY>F3qZqO530rC1?i^@(XjYw{AIp^TCh_Q%Yg z8=pfLtBABadva14QN`xHf`t5a$Z(T@OeMA|iK`!2amU&4crycL+D)Xik6Hws%6FX0Ls5{)hqEPeTPm4o02)Ok5%ug&mU&Qs>lO8rS z4I|)TCkHX6y15TIAHinh-LoV)Q36{hXaItS^ci+Yk4o#Vm8WM=Mg88(ox*6>E21aF z&+3*dGe-WH9hEfM{}pT!At3>>kpQg_Bgp!s(8{OHU}x6m`0bjW(r9Em#$}Mp?;6Z9 zC~KS}_T)`yz|f$C_r42%;22W;KnBbIXFf=Yio`ZJtY#IzBpG~N4uFfvmPZjn)CIB@ zqoysZw1p1oZD-PwKVmV&;SaXv(JFTTWitF!xERiS{od_{y~xu&0nV zmKg(FkLeSaBJ4ib!XTF#miiS-I3h@Fq9}~rZ}loguvHG$r+-5=!KtC2Wn$_|UOa9o zhFha9w>szI_Oj84bSmnp`s^M{QdhFKVN8>1?+R&`UJ_Qz-{9ZM?agECX2O95WsPBg zq&u%Ke_IJi;wjuo#g_J9)qw#cr`v%BqcvQ6K6rKuYK7Or!88!IO|ap-4;3Xteyh@g zH+B|7nG8J)c?!jr$6G1u3oI(!*sZ@;+e&sxSO&UHV!=dH`DA11N0emUGO`te=6I3k zPYJ_ZPRJa=kI$Ut6ldPKBShI!#D(ZOY`}205$x2hxgi<$kcRZ%1u+93Jx*qqD8pA@ zAZkd-u7SJ1>mHL49V}kRn1Hi}FfMB-)~2&n_=a3;3WLu5kO=oElbGnsxx*pSd$oc$ z7NrVWs$uiTM_kb+15N|OxT@6ZpeP5V>P9j$1MF3q z088v3un4 zCz3BG5k2;x6N{p+HT89^vUu?0gTFIrW-~DOst&MuujcXOQcyfKn4^({wBPhQkv2A$ z-C%Df#1%R=%A=nGMi0u0)FX-*Ry{uI(Xa3pzX^06xSVqy+)yZXlxmb<*2U}gs^lTZ zkPduh0=mX2v`U-eEJUvD{$hY{GWKmsw|T}Lo!M(KyLy}?Fg)!-M@ISLI-B`bxUR&6 zuR>02X@J(~NT|(T%HHFOUyze1mGaM>{%fWo2)6_eL!byaV8i1|zdyb7VsL8;To4qE z^GJty+(SanF0(}$BRfKaNE6R%IyZ3#YBuuqX&C(Zs%&q3vJYC6nUKR1iX!>%#?7EG zNpLf#k$nLyyG}Abuua6T^CDkH;nf>W*15h|2pB2GiAS8cqPaKn7cVlj5RNC4MM^yXV?zs(O7C$LbqdDRS=Vej}hzQnj`Ykq9Du(k1A$ zpu=zXGv9Y{&vt`pKF~Ru%FT<-$h#fFAr)pK4xKV+*}gK%6ozR~h-=N_7=GicJLxsB z${<>m%I1TTy<%jJ+*$<15De)jIOCz^dk^xblHid;8 z@kkS^#vyZ%3ze@gK7RJ>WA61YGg#(m<5UW)v2!pSA=QO^BLpm;mOJAsredd5VCV%L zCfNy4!+_WrX(DecaSzj~(uX-KyOAei^?4S$hoIzj&LDnxXiz+!7=M(~z^+KAv|ir3 zK|rNEK$NoMR;n|7af}ARZZjHEWLk`IG2A=^HxOYq+3&h)^J6pjyd)Fc^LCyoN%HtS zmWv0+qcX>ViYdjte_ole*cYq;);Q`512#tLCAU7)gW&zzn>+Y(Iyi$7?{1-qe2o9glwOZs=;eC}j&r^h`@fSh*_mwDY{ z87SQ?Ig2melR*W%aKE5~N}+Lh7G-6#?&=WYY%etm!wJQ7JaW1DfQc56v1P7iWS#X)O>6-JDp}S$pZDk{{koA~o_mgWz0>{u9GDkCAi5`MWtwwo1it{3YKTAnT21Y>9Y35nLtv0&*cn4LT+-_+A5wTk51H#am#BU9P7)rqs$_@K@!~&I$Lkwv7;+L>6T;t z8Y;MWTQ`zg55=ousU`U(-Hdv$U7nj~sEpDPA`)C@J1$O5OT%{vLXs<|Dns7e&;mY06ZOm#=(A51!opE;2YNs$k1@#@#~gi{gy znhEOLh}D-$0fBf;&fx6NNTwTUGCJV;^V6(2^7TfBorz=TqDl7AbhCpFyT1F`R9Q}8 z)>zzUJcZbWj?J$o3sjGGa*XVCJtULxxw}-dG8PXjU=xP%bB06p?b66e+2|V#+j-r< z*9gIm{WO&ET>q2MFWilo1=7DIabC!PA4!p)cW;h!EG8VMO<;($s3?|vPrtHJS#Jf1 zcJ|RUc&6}f(OtSuEf{GywwSG^g=R(f4VbbuiZfPoq<`_(=Lmi!fQg6tElG0aFx?Id z_I?-wMECN6?C^2=_NfCO_r_7hPQSmsPmal3LuJe!MDBP)XwP*x$=oNyM-LYtpW<}r zFF3OA(%p-fU~BJ3Y=WsFaZasy;o@TEc8L}ma5BTqUQmB^0g2%)1Y)WceXILh>~c@% z^fw07bEx;xlN6z|()hTz&Pwtze-mWFk#%jl^gcxB68F)nD&{^@e>Q3Sn?_&SXNj8$TL0TXf85 z2vPi&7Xi0C*YDJ!G>TCxHBdnYM(nO8`Ix}qE6zb8c>M=bU(sP<%8pl?_RWsVzKN(9 z&Q<8JIAAkL;g*Z!@`eZ8{0UnS>YLqh7X$}!ZjK>KUjoc_@;IfrLKz=ySIl;p9^>YL z&mw9=6m4!4Hn9nRNRyR?HOS;~lFEgbx=Uk^)Yi#cUQp$-Z^?YlZVOD?%L3#ghEcBScZD`sq@CR4Z<=U0>lvfn~y(n62kMbHf-D^Nof&ron%_ z1MMCfv!YnUP~NWoRv06V9W5pWebok}uA zKD&^iX#8HW)w>nOa5MO&p#eD}_dWa`3JB}qSbGaY5eZXf^yL@U8wV+ViZmPe5 zWc;i4>j>_AP=sq;cz%u2L2bERdUoc1La*CR(Lk=rn@aJKjXaawrT_B5@iV&`wiDaE z_r{|Q;8d#y2iKaa0f*0Sv<}M`^<&g|r~mf+M6A>0aA(25e_LNsC#U!QbMB`fRO9r+&=j}uzt#{;Xs@AGk|bzWG$yht9!)b>c5nt+ax`DMn5${p8t z9$6-VQLG_ttR7U&$gk>D$xE_JoF4Pw`I_H(V8%GEgTaZ0@kpGLDJX@`p>s1;K0rcG z@i#@PV50u|frgO@BrO?B>rTQX{#sSaB;C&)*LfT7sYA`3V-s7^n4~VK=aW>z65YQ#E>WJqbnck|m;+BaEC?UYlI1 zscp{f&yW^K6GImB&gk!hlmhuGVA-}8Zrz|@9xVKm%sto?r2~yG`sYq0nDm(FJzHb) z!8{CQZa@|{3e_CnJp;=I8SwW>MM+*u-FE@j{oAf^Obfic!t&w11_H`qum%3!k0`Qa zzBBRU4N;pAzX6co73mXQOd9ApjId>@GbzGf#Nj922{+nTf&6!V*;O*p6dOYl$a?0+ zwsD(y)f0@>YaOyh)v`e;*}K9Fd&HwiPa5@%|J>!=ookYf*H$QBT1q9nMW%p;!#>W= z27+4E@^%FiZsqmien zAKssmP4VSiM%?!Rddn&U?Jkb*w%bHd&K|}E5U$+0TbnEf%%vhX}!vU3x668Yn60}GVizuBA$Z2nLer^n)l%z}7sh<#j=n8Yj4`&evD4W$tD zT9!1>NY*IJ%0ziwq?ZcX#uhh@!-~O|BEsE+tn(&iC1M~)*M_JS6p$h7{~nt>l$LdZ zuBYOjL1?C}tc&Eze~KM$_udM{ZnAqoKgAi3O~D|m<*#iv>5 zQhXZ)#E`NI4!z`?3(#Nk}jIkyF@h<(5Dy( zo>5Q1+Q zFi*%%qUTn4dYxcO!iUlA5UPrQysw^@L1IaE<4R8?VzAgaRPH(=gmt%@w_V`2UEjdJ z-U%@d0l7_Qx{>$&C2@Yj_W(okv_I);yutrdQ#S5SQRIawqGVi**(wuPY@Se;FHZ@U z1JMe??-;5{e7>8Xnz#>pvD^Ol|pcB?z6>t1yceLD3P2;*(nIXYR5i~0N2GderU+<5a&#Cu(BqjW6j zwTcF=|I%IpV4K<}A5_2>OdP6mFip`DQfSx%Eg}P|^Q+A7?t_5;05IdpD+F?c0*wp$2f-F>MiSfy1dzn(x$zR%?01l|$HxjvQ9 z!F7u9RM)YJ-OmL%f?CW@uu#6aQ&A^loVJP^xr)S?A>|1u?HBtU3=NC5uSX>11)omi z2iwJAR6MI4ur^!E!Mr|D0IOYy3tDSx8#0$4Mlp!+B!8${sHoc~Ui5kJZUTkj+lHO3 zM-Ih54!h=Z6z&wRM}06e4gIfs2#E#V>llL=`(5zY5aHwt#m%q9?=(6pp1Pis9goX* zKb8}ZvPU0fHHFJDiEqIO1&}MO`(@QfV zKXqvcPoZXcXNiia@A)vC&ZG*ex!dBQ-JcC#pB=snJ>{1l%kEkkJgaVBR7P zn=&Z|o{)!8r(th}c(>JQNf+u*Wgxhj#V!0Cbfay%M1mB2XmZ}b=B&Z|89z_ez9ryT zT4LlN$@~jhZASFf*W7vVQCW+mjm3M@RkQUo8Gy8!M##JQlQl}m4eX1tMYsZl%pZqE zW2jj<5l*4LOx7^%WQX|UC_LNMhZ~9>lZy?%Z8~+I&(sND@aC?CN%+{Guz}wJNR;c) z6EQ~=PUIWxi(pnw(LplG(1y>&kqo4Zo(;G`(AVGEL*JnpAt?4Ubo z^Z$tQK->Kz|MAqnOX|~7WKr{!qTb`o$coZlS59c4IOb(ci+|&n!J+kAS!RmtXI4#a24nB~f;`gzc(ulv)IS;a|kc&wp zxx(@feY!vFb8g%JltTLl$tF@mpo10KtF{21+KTD$quM-E;fjyl{oo>jFu%GhuH=I} z)bmjr1_CD?Sc*_bv1JD5OtPz=Q`=R3BO_AdqPY0{(278oyC?nf%ms>h8H-zkqoK)X zt;~d$Z9dm*0(wY<2+U!IaK$P)%FXabX9o1mqTY)MkCi|fF> z?Xk{2$}<*TM%HyC3a=L=&WvL!@oy6jGm=xVrSY>f zTT|a4#d+1?60-41o9BupX4v$9Uk zR*0V_JI@gdh_!S2v!^*$_3NX1l^sf$Fn7X&xS!01>A(~L0}6kT%QRB?zhSSJ_GMRQ zUU@cS8U+D)AL(0s|B8#y*j>OBoVTt3Q zf;`RqwJLfUPLE$%ZgPjQxF6_l8?PzdD=fc4z8wGR1+VN61J3(!9w2E-f9DX`iXdRwZa$u;XL>=X4qOcY_N52m3CNi1(^>QyQ-cZj z?0rnBn`$u-I`g9;3?P*?3I;V``*4U4(W**U^}kb01$Rdhl*2WE6mE+ghS3ucmd&gp z+uaq#w*itDeFmbF@NvGT8p=GiV%a>REL|)ju?NTZjQ>|XoX*muQdn1ek>)!9LsjPT zl#fqMaxxX**n10JosKDG84oeRg390I*l0lYx**!0dtfSVR4i_l+{vFDN)Xc^5FZDh zXl7D}-jfY}NNHP8l&Z~2<1e_SpZR@e-}9zOG+-6hnQc_Qv?)!ouj#jiHD?N;6Xq~` zpm&BVB9tJyB)K@=0;u6M^gASz#KO-BO|+ozU8w6fj*arXC3OElA@TT7SU7Cs2;u1H zAw=%vcDOi>JCOE9yY*v%UG2YnB-Jode?}o(BH1zvYdM)m%AiTIAr)K%ZIPksXx}(XfPyt>%QWBpNv(!}DALc2zG1GPbL#BgIX)Z)4Lm-ZM;q7F>%_^Xftqd#|sJg4ud&QT&G*Ju){oXFKVQEbX#fM;=@ zn8fqf7Y~L*7WXIH;Qnu~*XkEO@_oh9z@U|#6iiGfWh&SRj_w5QYrU5KFl6y(z;&D@ z$kWcb#r#aJ=hX~JI`b`0j@X0$DQ{n(GG8~!7ud3hx?=N^PnZY3@dl`o7eLKH$jqc!bj-W#h})Z+k59`c z)K_Du&6*!?gYAb5X&MY-96?WxZfOhTLtNYor-KrdU4IN>A_I+bXXLK{C@+;2X}(2Qv|>0 z|GB+4MCzhn{yEg>44NX3INS|S!10~^>A@_%x1|E-6#P@qy3LR@Nwx7xmC9^90QE1E zC9zr=Q%n{)W+W~L0c1?aaxGh23!0<3v@_6B4 zMDC@$1++&5xucu1*(i9a&MSTAs?L^I zT>}DYHC>}8kOXDBB7&gYy6~v1{+z~VE2dA`~efLK3&n5EB+?qLloS)u(tZ}jdG6->vgH&~&)TswE2TZgBp^F2#DC)IJgGWXY*mc$x#PF z>BT+%5HgsK`r9CHS?*)f38()ddK%abkK}E3JC98FDqd;AgPZU^1==M$$%||{@avjU z3KWdL;B>)`da-NTT>_5$X7dDZjgrRywA4s}H4q`vR`*mPelYCC*YXy#9~$AWCkH+@ z0cCDmiw0GZ=X(g7!+`6(23mmuG2;cRWDOkc5v#%2Bu-|NPR|uy;Deni5tma2F9Aw} zVpDe518b7bldLs_zQn=$QG5X$K;7Zr++oas;X;)cMoq0i(|@dEeJk_7`<3i9GI(9p zcj(d;8nkzl^y=Tdh;^3*ePl+sD|XPlt@*S(vk{l_e}1V6&6eKE4bgUb;9BuIA<43V z=~M;Qufe}+o#wb$P=^mrTA>eDgCy7RGvo6a&?mfN+lj!}M-rQdmmPqt!z`8qVE0rj zAWw5b4)jak!n~7Me2NFN6e`1+isYf(s(PAY#B0X@h@bUiG&(igE9VL>kwvt+6!k7A zCh!rxE+;K^yjMcdr9G1->9x!iQQ9^lFgd6rBAk=!$p!!e0Fs@mCU-%6O@wjA--Xid z%8GQ~Iy=t)iM)c$Er{<$ovA#L_LgZZ(C^9>H2zwN4C|m6>L}3GL$57^^S4w`3kod! zNU#Cv?m(L$(AxgEJd?Y1mRRPdGJI11FmH@Wr61|`Eu+Hqyf2}_eZB4<1P1N3pxeWD z$6pDNXx%AXvwnfc6A^!v1rX#+9D1QP;NYMt)Xfd7v@Z?J9z0iKxhT&Rnh2P`4mgcM zZPEJ! znR|wCEGD4chvP4a;;rfZ?-jG-+w@c)s4#+YvwEK+iVE3&TGC8ON(F%%tuiGux?#xI zi|L3;hK*X_Hgd{`wqgnw0Q4u|5os9n{=6w_Y_HEwQFb>^c?Ge#P{E3wu3i=Tw{b5BP>|AG&q>?!|L#fBQSXcu27(z z4IbzHc8d%fi>@3pA0NTPyN@8}!visbsKS?OV4rBl8Os3I^~YG*Lc!d$pmlO0Y|~22 zXw>|=;YJy7R+s2NFQ2Vu%rn$-ktvqui&2;T?`$htDy`GYTeF(KUOJ5$k9>MWCSqwEZTlZ%x2g~F2eO}?&8s9278UEPDC;* z!NTH}i|XnCpmx}Q@lp`J9?eJ%T75aPLA?0|{f2bXn|a2HU>z^lPzzbgJ(@6R?R60B z!~iB<-?OyRrOMk8uk^}+!`QsmR0Q(@!>PQ}a{+KvW8Kq*LTK1F0M?ia_@oF3Cokqe zq%Jyiy8C`3jkmgg?p*6my|rMmk3#E2K~_O-EviV;d2`>@I3d)VRD+}^n5Lr|aZuyc zzCm6|MDJn-VJSd_tunn9^dOc5pXNV@Mi)~5XXuVh7&=uJfsWTr(L^bd8B)MW?o>k7 zZ6Cx!L%g^vxo-m!r4RwU6)KBLW$p+?CNc$VAXw46Avn-O{_hA}@c~5i42hzl_NZA^ z77de?ad8Y(o4GvrA-;}&tZ5XEn9B2ZVQLl-s(e@qnqZh<_NI2~62>c1PsOMA7M6zlUv?qn_FONxl+$ zhhjT4`7#@3MeNip4uh0g`l+&1k(TYXKysz$n^-5?vi!rj5vT{C0W z^yfN35m^gyZakr327CC1jgmm1AZEL?KMp$J9*jZzbX_Q+9--0mKCStR0O(a*|Eoz$ zTj$nV>R;13;UnrspClzr&pxmqi2k=NjFXVp{l>FM9Qq?5V^PcqC@1A;>$5%64>uKb z$lYkuGou0hoKu7u77Eg4cqPqWiSM)o#~gntEfsKh#V{l2oo&9}s-81Z`bp}$0*it) z_uYiIFvffQy+WQPPT-VA1H5o72`zw+z(S7AT^#|QfbRWfn_KA)+9J5D#jIwr0Jm;Nu~3!$4ZiGDb%<+^>W!1dxGK7eEU{1&atJB;9Y3VJ7+yoLRj z+d+m5K8DwybntgVCJYd8R@lrp?a9RY#4rt&DN;P081it8C&(A0yIw>~r>Rc%O&P-n z0-53hTAPzX0cc?7lBzf3#6> zh~$1tNMDDk7=LW1uHSc}A6sXJg(uVkZ?FxOX?T+<`!@y{*R$B6I0b7c>D>a~kfm=a zq-$W=c^0~Dl{930K|J_jv*Eyv*yfTWf@*=dysVV~2LH+9gz#C-AE8kwCiU4RHFf~jrD$W_#jaiEG`j+-$m&@-k z9)U2fi|31+3R)5^`w}NeGJV|y=;J4Ca}|F_ya8r;U9~;OoJvKRfCNUbZRj3L$zL(L zjJ+gPp^FoqC;``|o~RNg&6bkX*~e_{kd7;!sZO=cW#Wq*3AjG^T4I?zB*P>#m&T4g zT-4rwa+GPa%)Q3qZEw4o7hp$d##R^>H{~{(P?QeLx0)fql5dWkJMmJCkR=`kMGqAW zhUEhcykvq^tH^&c>|;%bsR1LHO{^?8JIG;!HyL20W6O0}zMyhg0npF$yu#dH~h97=V#(kcJhW2-~4%8I3Fc&su;#3L66G zy~&U6a0Wl=-+C~4vWjIisF=&R?Vp|c#9IE52^uREu57n$TF5v0|0{$i!QmU@G~)`| zxC>zM<;HE<^I5L{i$f_bD~pR@oc2tRUs%Ji6dgwqhxGs2t&CN^Int{%h<#Ea{Qj&_ zHs*y&!?0Td4=DYWDB!q>6Ej~du4CuXP2?bo0PJfaEQp5`oI6ldkWTXi#o(rc%pB#9 zZUoQngJ@!X&z^-5L1cq=nuE#~r~b}@i=f&Ir>880EDa}4=N{|eGOigjba+-|#OyWh z*HdkwE0B2AgVnZ~`}%u?z@QMsccXsy7(ts8FBtIDg}QFsX-Uv@*D_>?m#R-z5LV&< zxZC!bheP`p-?#??IVcTLSV=SX+xHFc!PFudU-rFnd*#3X&7xm8=XBg(F!uRYWUBOVH{J9gn$A;4gIm+u-SH3E?St=# z(m@Ix?AtqG*P(xnr01&S>~ZwgbGVQ`FS?U!D<6)Q{d%L=GO?ACh8^IMlrI4Dp^lq#diC^CIrcVL@LggM;68Y<>Gqu*^aRoP916m zyhk^Bk6XN=H*qYK`Gn>YK6?z8Ntxh7v#fs4E244K!}+uceluBcs(&t~4G3 z#7GK`DX2%?S?x|0NOdBikyh0OT2Dw5!wkVCwHBY{<;9%oYAj=@Dz}+95Mv%o`w%VU z^%95>({0w2WxPV6)%l;(SgyuA+S#VVa}wRY0xB3`^b~6Flvt8?>jS=s@#P?Xj7iPR zk-M$wm1%}B{1Mb>CfKo0t~z&j)o|*SFq*9pU=fH7ysk5=_$fT9DXzY=$N;DQf#f$k z-N{vdQ6aW7Is?Z#p(TfbUb%!iVH+xys2<)l%&cXo))m1sR_#kJxBkqTFm3-6hv=*A zlHK$2x!FYHiyuzdo%{hqg#1aHyz&s1c|LYy_8^qW!jS$JjbX~j8otSoTXNX&`R|ms z5=Q^D-+0BNZqVq|1QIa4EtKND3k6|gXS(WZSnqZX47==(rqs1MD<^-fGSnCy1++MC z3F_1uo+C_*#9OgEnyRzol>y31VfQGzBZ64C0u8V>kpq$y4_IVx7huSAYc*^McDjQh zMN6FF>y{ajFdd6B4ZWwtUM$EYp{}(eTZXp{XaMhzV;hdYeClHbeckJ9m5*QGB?`R+VGA|qZp=4Kr5>})~BMhG&?l63_S(Pub z@fi9^)xYmekxwRa$|3xn7-GengYjyo(bm8W3|xwX2_r6DcUYCCF;y&<_9nF3S?Q5A z#C2dMc-P}R98(c*g{$(Z`)r8!W~e)#)-(@VGXd8wreh|zO~x@__6PdH1bXX!#!e-u zkD)peHSNse>Uf}@PuPumwD~n;fs4vUpjCHXqf*OI`KKQ8y}nEvytowmaYT0hH`3y! z-4Gd3)FkuD_ezoPQ=`UnW8gEv_DWZc30ee60JBgDc}WPQSbr4Do%)a4*=_kmOv-Z7 zRVx`(=s|4c5T!O`aQC$UF7_P?>+O=;VY!AGPfQFy!H=8oUQO&NVu!xe`ql9K96K)5 zW(EV^hl8TVsF%xd!l103?YQxmt5OjQJ8jZJg`v`|u;F*8+q+NpIuDA#xoCKXzDFEN zW^XMBrtV@4wMH!-;bj)}|7P{7LxWhs~sp}4HdJy zsK^16O5!mH_&Fy}%KjjNg(X~H{#Xa>1iHMmIPY_|_!X~uM$`s$%Gg$_8^37V&XIPE zlL8>?{Xd7`&Jw7IQBxHMA zBgP>iPL8D5v<{Ie2XssB_vhp)`b!ok7r1--QaQQOWITxoQqYXNbb1O`@ZJ)u%%iZrGXS(L3fDiSBg)bZTA6{(VC?>@= z!5Jnj#Pm*ii}1KpZByU0yC@AN_7m_rt8NWFK?GOKLV#Z_^xTcDR=H#~~ zwFx$>F8Cm_e#u0O_Uxo{f6iWYsXuNRE&6_DaSr(w_8`j=v!w`E7-&+75`)u-=6&l_ zM&gaM7v6V}V>|u@rSGCkmgya?@*HV>+hD`S6w|TlBiL;$7mU0MRkSIX0p@fJH;F9f z6c1Q;B;Z4QZT>W{YL4t?>VMB`X(UVo!hAb| zdUV7S9b>ufwxM%T)Z+Gprpi#KA3c7^_QKH~jK|}*L&p9UM=Cq*g=ns? zu@{}7%dE;+bHDfvF6J9%ma6h@Ln{wc!}%Nd$lJfoO&4Wp)NE9!NRA!d^1{|Bo}42Q zIq-@w!o=*wnUpt^U6c)$ePHhqd0O~MRbedI_a)16@658ZGuM$pKn59{4+S^1;(C*7 z<^`XsKRuwK5z>N3o+>qtE4)jQZmvTc_#*7|>aE+i$Y^$24(a6HnV20Js|Pt7jDJXZWq#Ra=n``w0_N)r|nEnnvcJewDbsjRk_ z!x7@=x$jeMIA?>>*2kb8$Ns3Uc2-kBw#T?BGQAFxBMDNBdA3vH_+RbLMNcRpMPNzl!MkQLoe1OvTO^p~N z@FV0;>F6`a$c`PGTK>A0krI4R;AOSP`s_X(q^c4JX4PsuM65mRyBrXRn?inaoKkB~ zUH<(%$TgFBDimfTM(yyL()(xOh`Q6LrkeN1Sa4O*!kk@0=S;6-3FB(Vt#dQ$X_J&J zdm)I2AFVturvmt%y;BK}Mirv}Ge|@q?^e0JUyh=!l{ylG+~#+8OgKMs9EiY$!2Gp) zp0UKN%<9bBZrLZ>`e>|7RR_ouGIqhO*C-iOP&1Hj~F z?UVl@=LjyR!+gHW4O4@%SqHn{Kjy7U-UZE`kixBhWic6x#cvOSpEgQ^@yMZkH@?i1 zw3Ks>#_RU|hR#vTAi3iC-0s`AG-^rva9~>sB>wn*I0v_Zq(p%B!+}>M9n}l$JDq9_ z-u6{g!16v>d#ihYYFB;@^pF&+8BtLFI$5aU2WUhwNcq|e{r3rV41Ja@}GKL(sALGlkpCZuw@#C zV{d~0@wsb17fNw*twL8)eosdWGJ^qSfyQ(INkF#01y}##-FBuk_}sPH>SQ}T&bBu| zC?0qE0*!*HA*)(2x6#Yu)EAfnE-Ej6lU2i17J;+PkN3TcYfBC`{dZ2zTj!kN1B({g z4LXIL<$jk0>uDQVo3YCx^7%2Aj&)p6TrM>plhDS{feZjiK(@bBbljgCxEucxvo>?I za==0BD$YXqKe?zUE#O~1ToZZT56#uA%s39*EV7($n5~_~W|uVA=c{9Y++Zt0ME@VP zkX1s*p+<_BPp1IN>r(G>?UFTE`!1w0sutj)U<}Q0LEj6V|GCG%m?q3JIN39Cyc4g^ zFg(Q@<;&juFI6hQ1VaD4hXz(or-SPlbLEnWw#fdXp!>eisEHX-N*m`TnpCrmiL0M5HZo7P}+<>Lw zqio!*XqVCkY5kCbEBX!=R8G5)61zRe z=UQ%-D(=aeCp(eZ4d1yRPyXUVPb8bX%s?uB`=WYyT~QwUZy|5cks%=}RF|?oJR=Rw z@`Iq7`?*SMC8P?|W9@56^p)VB--4x5r#tzTp_og8Tg9T#a;TrdvqYM%8*xJjOlL^M zUn?U#qmv<`ZkG$#C#S$Iv=%iXrT~_~yhvoZ190u{F(LlPcHtghAo* ze}xKJ?OrT!EcgXdsmFJIXq7e=-qhUU#CMTd_W3q^wyT8ny5n1OA$Jj3gen~|E~T|O zv6{TEwJw6(-Z6F=g-4HgB?4M~Bhjo@&M_Yl*-5vGXtN5!s^+S@;G*Ptj8FxOzOQ^-s3J2BY@c}eU z;Z{JDWq9{lBcni==i}l$yjWoKsvwY622J}kWF>Upn63A+yosE9pk)o4$n5cb752qi zgSjl}Ws|doHSmNYrls2w^l{xhC8nMA4112x#YTcCXY9pIV3-+v4?DDE50`(BGtS z)o+GBExEM)fy8UA(0TbLQLVM+K7j}w_eJma^#Xnndy!3Bsg4pd38lkB;ksQ-?t$@K zM~vP3-U%U6xk9Xq-k%)TduHO-oCp^)uTc^P*Q2>-RGs7%Tmy2q=U$%#4HYpq?E$-f zFPA2I!4F`5zPWv1MfGR9vZF} z74Ddz8qjDwW~Wo$Kbz>X_rqJ2HBYH!@#$lg6(|SK+X|+z(TqRqU92opHjCQ2k+$o$ zAL?|-{KI79++;N#QW?#r)O_X?V7m7wZw`DleFYh`RzhHw~de6Sy2;mMi={9Dp1X;L4Odx74n!wwG$+b;lXBq1;f;vX$d|UZdRNwZ<$hUAa(eKLPSB9Xo z_|+S}e!xA_!oI1>6h!F4?x3C>Ayq|HfO8!PjF%z{>z5fiHCZ-Y!~_y5Yz6J-Q(chd5l%{lmLF zr2PQ|eU=bEutm|CB8hmRZS9zqR1geY*Y#5SX9W`fW`YYVZ$Q3o4tjTTbbBG;GZ&Kt z>UoQ(abF+AYWa4g99&!{kB}#bTYX|>m3_zT{Y9!eO0Pp3e5*+S1I!<4ots^O$>s$w zX3tfrF_;J)d%5U91eJeH`x#aofo%qc;v8^1P>we4o$msr@;!Snn7*OMS$>4wh>jf# z0WJr2GK-1VUzxDD&uDP@@zjR^B84q;bi_;x=2tt>9hrts-DVTstv<+XPkl@ra7bRM zq8Bjs*9|t*+wWxp7HVAs1G)oMgeO~)BDpZ?o7L|^T~*ds_iK)xowwB1J$leLH9i*8 zT73g$=KxL1$4@%y)X3)lQz5M;?U6%FwV#`#Ikd%H2Kj zY~=`K3|p5Cz>BHK8a!-Qff60B|6CMz*K#Dk{2w?0Cb-(aF-VzDDPD^mZ3s2dyW9`L zSn?TUJlh--nb8dIW8fE}#4q(8G230ooaOaZ-klo8tv=TfiGH?D1i!;qGPRc~e8q#Q zA%h0;mvQ?NU)BPH3ASREPgG&lehzUn)5=%r*JQNDxiN>cJtQ(=)nRIyHm`5z&fHth z(sVh9O7;rtxkNEt3z#L&Px0~7vCSBf^#b&`IL%#npjc+!R}2-z4MW$l6M2EX9OWkQ z4*t#6K{{1VtgTT<5tvyG-8qxph8)|_bPhuTc z7`=I8*EWN;I|S#wak2-nD^+TY9!-?&Y%<(wW878*b5dXXxxZw8x@FiSa)dD7m;Rxo zg-OV7a)%U5grS%>9>p|WNtMFLp>Mi>t40`9@(dR!g_}JrbM-tb?~No-QvXakP~4QC zu^7cz`Q5PLK3_uQrf)n6%ObPY*N%>OmUIqDblVM)Wu0&|Iinzt4MDc9?1(1-&J!$- z4%$f*)ZCsPGq=N4{+ZSIU-^hzJu8RC3WPK#E%vd&RT@4r_dp%|d|QEO*&&2-vgf*tER<<&XB!pl1kYktDUp4Q@F zAv9I&TjRq(tm}_huultY_(pXQNZ{wX{X{>qE=B(ONp?dWtwxjkin|rBYsPVM6=n~} z-y~D>h|C8!I&XPfmXO5K0wscljDl|@UJE_CndQI}*2$j|D{tfE)aO6k>PIxGZBMP|q`aqStG$qpgN9O+cb@ztNi=BV75) zgHR6Jx@Juj%4-vfvk#gqn1jPgh!n|`UB3jLXvB?6}L+f z{i@92(wh?xk`k`m4vo)zz-yI7fPQ0%0hqsawwF&UIm2BbA9$x)GKaExVcRVHGDez) zao&x=c%69TAt@E1)ioMT3uo$$9F)37od|M&#laBkmVTtT5LYxSsb7=L0W9IyP2H-YOdUBFupKWJ^3MqWT$491-nUC{{6Q(Wo>Ctd{)Yz)U z-VlqmUxco@pJbOIBsn!DS~7{T;zlVMc;Vg>R{emz5fhJY}$7V5OoD%I94 z?pqInzUR}t?f>Xorl0Y5ulz>9TPtI@#C68wudadKp5BtERN}Fi*)7n9b0X}M zB79+#NpnC(5bf#gd)zl+Oue;Vy~DcPg>52#sTP!i01#*U$ur70AG*fU$9*(ETqhoT4dU`U8J^P1#Ex(&G8soN?BX}X=#1D8WR;vlDV zfq;dfRY|MAP8PjqfbsDXUwQQA9*BT|?8a)`^0&>neXPMZF6JE?koBP^d3s;;t94hL zwEf)*Lv~66sHBB`!weQS#Y0^@VMEL5x8G1`E zJe@paBh?d=xSDK^$6`+*UI?p4##(6KnGJ%H7;c;I+cekyv4YybV{m?;ySesCRO~bT z?30Pq(X2ge`Pgm%T1~>;PTB+R`78tj`N@v0H_o(!-BUoeb^z=r_kJRMfT9B^Zk6|E z1G1RHr7lf$x=LFq*|;{LrJwZ58f1YqIw`LS0uvc|E|Oo&IbC{^w9WQfc9#g2*qi zBLJm(@!0EXWW(_$Pjgmsn$W>{OSd*W4(ah~*#2@?4Fd+Sr3L7V3E)>}s zTfq7NwekyY3+w5Gn?;TI8-Ozq?iO#&KZt=jmVt%b$`%bXLHdbxlQ?o|tO zb)nUCC<#Y=n?JWoL7kBv9CaUdv>j1nF%;2RnDu=2zTeV>v-hFqi|tvZTxiy`#-VQ% z@*M4mRLwh@(7jtC8lVs>tkfK^8WW?7)z2h8v}kF9@}ZhN&6e#7ezGWB_Uh<`B`H>Q zd3!+^-9MeW2TEMxP6V>jg}eKPj0J@BefWE(bLWBX;a54XQi2xJbFLF7!v@w0Y7I7@ zH2g<46Xh7zjHOxbX_Oaovm%eCDIx=6*mf~^5zY`0RIJJrjJMwM6(3((8@(_5{GBN* zcD|X3hsN0#nWAPkx4EpsZ8hl4@BkhsuI^d5r#4sxU;QD6{t^#!0fpfkRpaYXMf*!H zV2-dtUmb4hv7L6<6b;J6%F+le@uyh+HaYSp;hv`CPF{Cae%vUXo!d(Mp8134sO&GM z!AuIZc6o@6b9qS}<%Hil66HDhdNSP@77pziFQOLT_Ii(V(6-W^^KzTH7md$daNf&X z14zu`#pUw}G?TOJ|v3HJVVnY8Pcyu3_E?kRgScgikq(qC) z5Sp&j2~^npT@{nwiHS0o7kz8BNZFvC0FlBqgBUH%sCI31Pd6#fqt4lZSRJnI-T!iH z#uFjQo-V=pbm*Kz%=BZ$=57M4V8A`lXGV<9?NZD4PL42XEr1wkYMMbIRKe<$9K8i? zO?ufTK%6}TQ(uXa1Tl3GfFD>MX-vJ7I0E~E8^y+?EnbY^0>h>;V(C8CCGvTl>D;LP zQMFPU0K_@c)cz7OeuD!*ksSujh|NCHqcEodyF_4=3i7>?^+}A4(`^iFhX9lBEaDYV zuh$H6B`O;t2-e~+hj!q``fSh$ljpZf=(JOLs&_?QbLWThaYlG?OH_yDju>lUT|(k} z7s(=KbGXDS6TU8Vu%(P8bgAGeBDpQlYefboS%F=$^^xbIR1w@nQ&3=EbwDutcBn@# z1|2Y0Iua&OP1Rs$?zVKWCK27@VVOuf(<;vS1J&p-b^jW)BM?OePzvZ#Zq|J6^s+6f z4h3p_v~z%hJP*KsrL#d1=y+6;A*b)84bj)jeDw0g#H_|9yuat8Y`A3L*@49-{DAA6 zeu*}O9+C+SH^qt`w>t;{XY}Hgspv=#9bx*e=7R)>DOMbXUc+fmKDMvLFXU2I!#ilL z*|ur0VO%L2u0Z@WO+#{N=bs}^BcSpxPW60wZk_UD3)pD{`6oE_T0#A85Q^(CwHyPw z0HT{$w{Hh<)yxSS<16z-#~)$z+IoX8_cwchzU_SpLb6D9f=#7iB&zS^2(`aF`n-!O zHj$*J=o;kUb`kDj6e9BUOy`MBCu}tpnmWJHVa1MeYv2DKN`uHDC$~sQFtO2h0w7;h z#z}tBrtSx?d)feqs`~)qA}AdqP&Ci|bo(OfXl6{?_IxL3NmMLF5A&2C*1d((!=7 z>pK&d(&@2y)nS^5y4vWzPWIQH?zTH(IgPcqs>0g$?X(1%t!Z6{lPvZbA$h&zkE*FD zK;bJSvH^lX)a5z|505S@;3xk&xu*Idqw47B2@qi2B%fyY3-ODksQXCU3)(M?g}s>=GMh-KYp;k9K2vz4kU}37`5S_i^vi`LUcSo_^{1d9?G9T!!3O$ z1VarDAx|bnLLq$Rw8S}~FfW3wSYZgqy8WG5>gw^NQ1k7DX?`lulgg5`A&tx;%GK^o zjzOqq-MjG-@q(%Pdnh)5Sb~^b_Y(HtB!JG`_-M})j}Y>nPOflh8G9`i+;UD)sXzs6 znS=X2_Aq3Q#a=+J{Uqk_G3x{eb{wWHG+=p5LrcfGA5V<3${Z)$+1Gv3PoDk-jP)A4 zVzwtRM5M>67wb2u+SGLmTot54BjuT{$sPBDj$N7*(LSiD6|#ny8D`so;mt8h{pmO+ z?W`%r4rB`$a0y$}-PNIUxX5pcQ!ZqIDd53zmNV&uL0(QFVBbrdcnA{$!Bb^9PI0`S z{H;~ZOB}3Oj!CDNEAJmpLHn{WHb|F!XXocvZN>YBdLXrG^#`3w;B+r^Dtk`20iKmu zO?~&YA+!WFKCF4tO*f!?jYwpS7+E8a3D-vH$%la#I6C>Ik}j$9UnS8ioMyEM+ZJ3z zI^*$+$v-9}qw@?!JSOz_x%ZA1tZo1n!LBRKbodgoCMz}Agg!q=;AiHei-CzQ;xuSI zh_6S#-a;+evmbM5i$uTe)1*KL7_O>8Fl~Z$50AW_IRAG<$YAoGLM#sz7yJ7u<#7^- z)^^tsbYS#ozT7wJQ=D(~6hooo5X8zaL|s(cJJuHkEql|+4Q};w{t2hwhgDvq^*Nde z2>pq@6AWG^Zirrztyl2VAgJ!MYo;61LU=9(Z!{&(oLkNrD|XBM>WI6hki%Eh_)YU0 z3^Ll3lvR-l`dD_-?>;3sq1L>qF$PGnI6=fwr?etC>?Kuxa?hqoSdWqT;l3j<`&%A_ zqs0dh)1Miw@)!*ZVuteUHmK7;_iW7eNcLxG=KN7xy+3Mbc%Vk4q z0rl3Csikvwu$I;{JU`tVD^6%Xe2rN#k5o(pjLjB zNeMpZ&MwFaj>PIUw7~RAS`45j@&D%fVJ%6xTc=#via;lihB94h;qMx?2cv!Li&*Jb z0S;U^mSon&7ioYl6iH>-+CH#smIR0$j+_F~!-oF1ae1}$cS^%iM7Z3eyE*lLLdSOt zt{0@Q(80|IJkALrSga1BLqFgL&VtQZo-tZ8G{%nLF|O^?83YAJ%G>U@w!r&pUY#iL z#oH+cX5i$5$C&>`lm6E4RnGt_bpnV;Cp2LES(>uCg#*jr=Xim7))s`1^uLI86{9J? zuWuXnxf^SXzE|Ld4M=$PKNXTlvU(qHG8e>Ho5A!SPccO+Zj-4s+qI;^{I9|qQwo3I z>RSG)cyE`5g8JtEufyuc*DFD_Q*}6!Vkj)c*v#p_YH@;LS))nL_YyPPn%cX@bBM2{ z#f1@8QT$j3CRz{bkv1%|T&=4-iZh2Qj4H_uyXgCEu4l9!$Fe^^q%{s7eaEjp6ES}pXjSZLITWUNL$Y1>UVlc_*NJ1 z?jeiw-WGIO^zp~p;h8#V*~rAvB>>bMI&&$GOP7h{y9!RL>h7pgfK(54fRN{LLB(_P z2U_tGqbQ}_L!YUf=QdqJh43kA_6qQ}_Qq#sz+Cete)ZTy*ekhv&aFoExo8u|E040G z7N~!(3oHG~q>IV0mk>C%xln2b+3j@cJ<6SdCG*$Po8)qD zLH?yKxZ5*4kGXI9)@pby!Szh4a)fgW`tO{ssD&a&G!%oM)z1;Y8eztRz0AoB`=`TT z$I5z@RmdZ4!HAfOI#U9WZ!O@<&R?K44T%CaBVVXX8O(ujasfNgPORqRC6XO=07T-} z*U3Oaz_HXF^jp^ibYWGC2}hc@xxbon*HU~Pgu8U=t~*HMs=%B;?)>ng&MC)!xbw%T zfM$f205=D%Lg(vDM6Eb+ezyu?HNl0)-r62bl=D)L1exY%>W)aIzz*hk2}?Ylt87cf zeV;#e-Yl}Marle3h~7hxT0t~(#PV*SG*m_r+kR<@Z-cKkl%%ZAy3z@dW{ZUxou0cW z0x^YLl-WMu9cuGTB5@Gw6)H~MZw%%ID(hbd#vZT3EYN9-(uf(SoU0lqhrXM-n+y54$hcl(tJ_t&Bh;1~XW)ic2PP zGx}}3*CyDfo*KF>q*gDv=BDzS843=0dMi-MAW;u0?+w;=7UeG_m|kdp*>mwo9=-w< zjn_DfC@131w%pzQ^7Yvk1M=h<%R6`aD*bw-ggH641cLFFz_ET5MP06p4s{fTl}c7tDg*L+#0WmC-B4E11AU2N9d2pk;m252^Y~q3_ z@*uU|*+-?sqrnm#ZnI9nuE^eCRTr8LK1|^z@*SC{zRGePYedo>i=Ke7rwsQ7hI|+p zuAM}e`6FkS2_cX%x$ollyR41ta9xEv=)eJK&it8EXEF}pilZ|9ptJS`oVppmOl2fX z6_J>qV;Iu&E0^)^lpXQJVp3WJMv^+RNbB4G-1-+G#sp&57+)4HOp&(w!DcI!0i+}B z$J4r{vI3zA{|2~hE$<94gX6?hR1xm|Ci{15O_!*@#BU$ZzulXyLd%zGCFQ7!sKsny z>x4Kz-f4%jevBtIjo4RKP=F3+9&tcy{#0fUkr;Th8bQ4Fkb_b!GT*%E!-YKf3oxWZ zK!KxkX2O*Jb^deZcz#V7xTTnN$tDyqPepyx>t_G)6$?#e<1&O|t#Al{9~V4SLGUT+ za47+WsGa)sEHVWsc$2j`w(2uwv%cx9j_H%0J3`C^w%jk$EZ60Zf|@AnSn$tR78j)H zNXu^}7O2tA0N%#IX{PMEI>$KUqQa>vBXuaz^r1PS{4?V+iUZ!M3I6pX7wZ?VE1^Z{ zG~To_57B6+x>vp+?im4jnP**w&+(~TL8ty5J45JJy&$oKQc>*N@;#&oAvAC*Z(e9|2?oi>ZF+XMPykI$TeO#@x@@{6c z#Htk=%C?f~Io(^@6!Pz9;p?usHfut`*XgC9mRO#apOY6y(O>Jmah1PXIfD6**B$g< z>I6i2W`)5^Z-}ZAE;D|ZMnQI^p?H3IH8^bJ?gZ#{0eZS=R?FfO;Twn;DzIna<5#PL zgeY|bjkUoqJsGuJ?zknom{Xbhq=TlyZ(gnxj(w$Etm5*5WUa)3e(7GHVOo)pvv}WB z7_M@HO6J?Z+OQXG?*5vPUnU%rb&V!EfvD-~N z5x?}%pUi3qFHVNeSEj7r^j7{Nz!+u0zv&0cYXH-4~xWXY)<8jdd zdqXC0xg*Y_w0Km=o~|0>`g$ynP{-d2Eo1*=ZUQ8br0L>Vp}}8VgoET6EkF<7uC2l^ zOhNz0=-XvK7Kl*x|DK%c1>G>L8E<5T8RD41Yh&Qj;3>mWXF&j$ zF`u}Um4|%c?mzsv_AQnc&_PG!&cK5*CkV&FHi^m66+?O}co}AIm}6J5J9c|(9ZDf? z+*k|_`91(V!5V0J7VGA0jsdC0;9kFtL8Wj+8p~ydxa~sg=1eSrXcxjS?v`nwdg$7d8kW0aL<7NRfECV8Kb; zB$8(@&OB$iBx@g7=4C?w?)Vj`Bz$9_;{v!rO+X?AydxRbxc^CTig*TM;L|bXDT$Sq z^aFFm_Dr=8+H`;&Xcj%-^UkB}6hP|HQ-QYsG*K?p)6Hk1PONMH3`xCFKD_v&#Y;)H zfr3L=VebAao+3()n7rNnJ?nVQGarb51f6P0tVf@aPhi*p#;87do6kff#9|`T(C1iM zt!}yD)tW$Z8dQ_iKkKMGi5LHAKCb-K*&!(2Q-#!dy$D3b+4x6{<^8rKM%gd*&OI%! zyCG1=@zX(`fB+v$XVGBUZ%J=AawbKME%_Pp<^iD2siu4Q)`t=WYLvW%A((M**--1O zDB$A<0CEQ3rnNuumWcaZP-pLW;^wvy`*!;43dA@Dem~n8PHu zLTu7mkp|1ow{$Uxc)~x-*+Zt}btC|86WZj;A*hXr{=IZS!%T}!K}$H9Tw%ck;iWE+ zcPXfAs-;*xDr1%?BGIfgOK@pMgX~I`BnwIU;49rWjaH*=#g~}aaAfA{=K5cbmM`h; z0W>3Gkp}OoOXJI*=iB0ub;CbGTJb39uSc^_;m2Jl(Ehp0I{yW z5!#w_I~VfvD)v5B2!=fY7}5xg%Yg~mkS>G|wm#JnTbGWxsi4^n(?2MGVT~LXyPTg+ z?}=+paDVT3D6F(K_Oweb&Fi*+@awJ<%jQZ;Tr{2s_r3dK#m;(*r%dHPtqn35pmLqw z@Ve=-ZxRd{bXQcJ4mi5odw%GX%H9@{mgkiLRRW*I2mINzO8tUUB9G`>s*Yb6re5A~Gh8+(90POrI1{)B*^g zq0-RBK+GAIN?n@+9*Bm}wgoa+rpGIqUEFalx-tYm%Al@c^CD&5n2(mESSbQqRlKzH zH&cs-OHCNTv{^X5HBw3uApilMn|fFT=4A&{D(wFG?zE+^ToWs@jaEXW#jU02%k@h= zP$)pUnj6kWNCW0>Gmx<;$Lha>W zZAT(un|R}1Pr9*_li*o_aei&!uOZYnQgyCO9&6QCu!KfrdDf`3>b8^$hhWU{9ywb^ zn$kg&Tz}Jc2pW{RXTN67scJdUml(;nin{i&AgN^qmChcuVaOwy|CZZVJK7hDyj;c?z`B|}Gza{I*Ax8r*CB*& zu&yvpVNwAU6>w9D@u+&J@`EC>W3bX9$=xY-u@sB}kF=*vy4@TIfx-S*wf!GqqEh({ zO?RU#Uls8*xUje(mfni8k&_UNam(@G3;Nj3$k@Dx$nKc}ntP{gc5JF2G`ujWxD}jC zWlxJSzi#GOPgsuY>Zr}mhd;vv(8L+mr`}_59s1Higb6{mhW(rOBc`M=0J6a~8>|X+ zNZKxN&x=g)zr@5XR(rB;e=>1S5Xy_a1BevOI9Lxy?2X*fcV5V|U92mF9Aps!x!gIf zHAF%7vblGLh7ixfkPen>{8IT0BA(_Zs#%1_IS@L+VbcPa{D1{TofZ-R;Z{=$qqSvf zwKbHjiH6||#X>o{_yX6xM=)`jFH_X2x-vE)eVQ{g0tCLI2Vq8R4Q&FoQ~rh4Y6q*0 zqqW7;?7(05(t0Zw;EEpNVK336q|OcO8l&9z=E`Ku{sJO0x_6|$&VQm8!P;NZ(gA}n z6)QE+ZKYnD3*g1f5IYvau}Vcq#CgzM{>{Y0;v9#mt45L}KfSsUhhylbsQ zpgy`JJTZ^Abg(sv?Iw?c;xx^;(II4z6HHdZiUmk$RifRa6RhX}6}F<~&C10fpD=D) z6)9xs`8;g!2Yk zTb5H@zt7R_NOm$e#&0Jd>d*eSzYL^u9#%BVo%tQ|3>2(w)wR#65Ot<6#r)TC_(zcQ zBq@KHA_7EN5Xo8N0sWah{fkV?YNMXt@&O{;zbx^@*=&Xq_h?+xF3R*Jugr+D+w$<% zn(;Fv2jPO)1GqS%HU&OT&~vMSK6q4idV@XXqAGYe3k1#Xfoxqq4EnzmUf%fKz&pF| zw8)7O}fu_3Te1~Qcl5FZU=GBFylrV#;0cqWNvbZ9{&hnXNb%Yjcro@UX(497kj8v>R@%wa+8?}*ExF@CBv%T-t z#~%d<1zYaPylm?qqBt$l@jh`w4U*{e&sT}gXNMk!5$mHMZ)L*sWANsQc61y9=7dMM zZ4Z-b@W|gf29N^rM5l$XZGmK=i^}xFEi7}9m}>gQzEm$*djO{eYFjxH1}0<|*|2yF zm~5W~oZb z+HzX;c_Z1W_8>W&^5{^bV5e94=$uHjPI*xX8j}|ht0t~No|~1m5~rfx#+VV*LfI); z+ZdZ-0qOu=yFAs1dzuevwXFYD>)?*?GgP15jY0Xpm)jf&K@2>((Cb;V?nLMTWk%An z3dfg#Sswt;blzYF4u_Y!p2Mh*ML^{H&;B!Lmahi88!rEScx({g^Mnzu=9S89A>LDIdj$;OX_h8TC*G1Xi&ByAVwrliJ! zgge6SjG8*+_y6PD+cy@OYphXxT_e*gPT&bx!@}dORS)O_m!*4)#-x~`7@>=|Isde4 zGP>n)%o&XPA53GUET65hxkv6??0y7R7^Uw1pZGu#4?>NIys7l4K!Og+=^hq20qHM~ z!5;%Frl#w3s;guw8sG{LZsq6xWIC}qalw<8L zuL-JIR`g;SzAy0@F051^Chg?u>ETJHwKX)kg>uB@MINI)Qbv%bBZsM3z6qCWuh`0w zd750aOv6Y@g~;w_wq76}T(sPnv)DxW-K7_AufXsy#-U{Deo?+T6>G=lWm#^|Qoj4t z-yd##6!{y~mGgX0DkNGqg(P_BS>W*28?ZTLV~3}o!QBy+^T%x$^5xAHYcK{tg37nD zNP!hSKpqS6;TgCjz6!Fw>vjci{P4@??V7JymU|+{OpouMY9u|1C@@vkpf^y=9{GjT+1M@6oMy%MdjNr z13s#;>9U18*6W&sls=F6IGXMO8km9zW7w(%a0(NX>Hhl^<5yVT4pHB+$ zT13DcmW%Rjrg;Ug@FFQ<*_+WXw+&$zq zzw4a6AIGhg_m#*!jj1PRK=H5DbSWYTsaf6N+Lc{;9xpAEcT+D)KmOuNxpKa|rb#lo zQDjnZ63>XJ(?Hk0g}R%GFTg;w`!bcqmtM`<6VrLA&f|ERng4EE!@x=+Kc5#U7E2fs z5?{9@$C#)Mv|6bW3|{T^?vjC&J9QG}C(XtRt$kE*AnvMmF7_hjG3^GUmn%s%C7TZ` z(Nj!GSFV20tA03(5vvyV|6%Co>i&I0$33IvyYw^8(#tjhid2|a${Q|ymL`n%* z$af%XZ?@mncwcjQ`Z1%ZwQ*E?etmv150{I#%CI}^10TX4Ia^5ZNWRFY1ZY_g{a*kYZj7S086bNJ zK(YJc%o>(#kaS0G;gi|4ZB&{E-iXYj!vTIvRTsAgT+hA^?hAG7+iN`o5al>}V&lh( z5|nkEQ2t4m0os00+{O=Npt@tYq@csp%gTzzyRpvg1$pVqF5nU!N&Nz{3Ci{LqLT!s z!1Yye>gt$$*B{Uu<3qHStXi{{xx<|bStgqVcJtgKDLo|Nika825wTxFNKQ)(1M?q; z2V(nhb!6oPoeL0uHndAAr*5H&j3F(YE8qrB6Kv27BbTD*+!Ro-&o%Ew zKW90$vykgq5_M1(PH9!cJbh4RpsSD4PC*BTWwYF5UY!{NjUS?5a~8R+=wKr@OIZzU zrsgXT5s@$sI8?2ex2JB8n;b>oVFzuc&Wi6EhZ!_c^@Fzb$aE!CO!^`zH#PVVc1*fz z$<(+}$f&d2oDb$3Q1XWISt1%(HqnK@;aeiRxsoB{`rPOZ#SwFV+<>heNz6b!h5=F+ zs3_gq8jo=dtJ^#rP6)j{thZ)AfGOxaB(~V24-KG*qoIemv4o6dm&aYT0xKJ7iM;*# z*o(NM#u}fhq;E9~Mb40-lUp}_qafcCY%KBxom#JrDjLBaZ4sw>xv@_p#AcFd%NHdX z3H=)O@PSv9Xk9_^@0l722-!~vyi4%QFur*WY#AHoD*x6TBmx3%F%uFZ6h@!qgSlA|InGM#&&DGfQ^Urlryw%UkLT*9w`=KD@*M6DT^&5hn5f6~wdN^7 z2Og2mQyc?quhEAhTFWwTy}!JxZg zjct)1)kxW%B*HFZa&y=+xpr|E#oJc6B$E&EZ#^bpPh7+Um*$afBR)=_i@+f; z^1?+UufVy$XJv7y2YrKbPD8hp8vFMBIafB+I(Q%WGQB2Z(AO0u*5LO^mwoAq4sid3 zJWPJ4*a0xLd!1Isb9+ftAzgF3Wul&f_2W(eVBz+Ly-8bAHgsBGNbsFq$zSPVO=UE; zRUyV+v!AWzu|bj*&P6vazcRWlZqry~G z9rK7F0F|#l;1-$?0e=5**hn0D##38OdFH37y&-AS&0cbR4lAWxWv2!uA4Ajcwo$^+ z9tmJkpki#g4lT!6)}7pDWsMA63{HIn5VtzVn_&^WXOf{PITi86zy^87&!loBbDb5p zt`_a|+=JSgaW{1)Kd-RA6OowFJiB*A3>{K@1RU|ex@WX$vAlP&&)Yl}?gMqY{Je&K zlu6-*V3zzpN7FAhBC5{=a=y@*wqrBBk$jlNkG8Ve%{lqVa;tHn0YcdLR?1Xp(U?^t zX|($BIIRc?*~^_V0+=3~=bB1!GZ!co4K({(yzJ9_2a@W1Yakol7sEkULL-PDj!R|o zhSYoaz`Qn*#nJxAn)LkVEfNg5Q+nS_R`7rt>>wSIN<48^)eHJT1LZ!%w9JYKI?EuS z1N!9%-<92MtyxGTJxlI|dUba(pU=W-YW^?I4$4X(9K*PYuju5Dapp9;6Y%wg<~^9a z=QE6*wP;9&TzUuKfmgkY$q;1$8LaX7&lxTWQ~+oJ@VWtp6^U%2$5K*Ciu9Lqct{)fzAnIg-gJ9P-iQiVy0+Bf}4Tsfr ztkkr3r-EZtVzufnqc0TX%9~hB+rxpf%s(^*E`^v!sY5j%4k{GI;`C$_3ll#4#{K&) zy`fiL)UUga(`sp+u=vdPaE9URuie%aqpgn3VyX=sYU(YW*zVoO$=Q#yCc#%|^LZY} z&NoN~*C0imFMO$5W6GBm>U5LzW+Uef1V9VH*1}J?FSah@(Z~-mqvnwh)6tkqLm+cL z>t$3Q4e~X?#aqVX9%=!92IQzj37uhp&v$N?xrZqF*JK6S!N+h&2p*RLNhSsXCWmQwfL7C%94@Qi|I-5KGK}tzAO?d&V=< zKs|YJOoKw&&AomqXi(|zd2f#FOxe*ZhoZT1>`q2s7)7Dsn*%tHr;0fWP|;qYF#gb- z7%3!SZMxi%9qEsb_Vz5A{&bdx+A{Wt3imd`Y+GLWcRn5X87kF}9^6h9v>sNoqdOn{Ye6r3m?2vESVNmSNUT#wOJN2N^8|*6D0Jo{7s8HmP#V{mi2;8=-KH*XhWsw%f4art=j@dv)x{-tM<w*iI0fj z#(56EF{#J&5CqHig%i=YIODTwYqcExMxCdE9kki{)0SarVi+pH586}R`YMK?002zh z_;yV}9-;+8>bFf(k?YGhw(8V+(vtET#?$_$M(9mpQ3Mj;NkM&^W>cXC1#tzq3VLbf zu*w0j!y!w9ttfD0YL)Hr@wZBb@3OJ+SW%wveW(OX(h1pgoV88Y5(#P8%O9^&Fpdj8 zbU5x*tCl*2VY}|?y|bXY^k->k-Owyq#%0WbkSE6T1+3NEV5JvhQK$o2NU%jhd!RY8 z`kS|lNAi`DKJp-D5+1od!~meaSt6f|Y`CMa!9n+bVEIj^qGvu@K$xwQp0K??!*xxx zoYMFo8ppCT$;|e@e4$cH{qMnUvRm^7GD!ZsF25aukH?f=5OcEVsmkdb*zGOHdf@%Pv~h9d zPt-T#*pSf9>}e4o=gz)omYRK^It&d?2E$8U9?@iLyZXZ2-AqVRNO$hp^j-UT`*nhg z)OlMg%)@GoCFi%q{)bu&mQkH(5pFGIzq=f=&~-2H9MS+!2c=;B6iGhR@`J8YLd=x} zpfHm-!WkX#`w=uc!Kd-id=sPFXr5qu{#EO+rTe#zd|MQvS41Jjc0lzUN$ zQ?cEgWJjHjY>h=~g=Yi+(e5)OKMaXc_)Qa!x6>EhzbIhJ(2zDXBb5WphLIfSgOB7P zy8svD6M3yiEzg`hN7f0rPKE_Pv{JCMQXiv+Jy0ERk|j|#88PVNx8Ca={L(zwe5J1t z%^geCiAzf9ZrG+alpcrD_fgEcRz&CA9*b4LP^jtO#X-00lPoK$$iB)q<#ff=Vvi94 zkd4e?3#mb>B#Nb-ZvpKuG9&&6H~eZ&*0Eaza~IFbY2_wFi5PzlO;? zPa(qh2g^2WDSkP5Ns|UO}B*ByG#LdZHy2tC?Lcc<;;Zb3ku9 z^U`Ola-zX4c9l4B5E1jkqJAZ`+0l72)n}Nxb3A}I!R;UmOWB)=9Vw5+;aMVv{Cuk} zPot)yK9j3WP$!U#^^eYwqglr z8r_P&AjyiyesmBO3=ApNfvCN!4M&NJEA;i~8R}|ZPGzQlJ|G-HND)kMiAF>->f*M* zmEs`DBmhf5w7*9W+i3*sER~XzuC<&g&mHlq3Gf3qCSqYje}ZA;ZLQdMg~3%C*f881 zy~I#vX^rVUq{ql|E(BlkIcX_MB5lTB6&USi)DfosMsYUa0*1TJ7)23<$}+CgbH0%s zCHW6fF0Gv+%SnvBJ-|iPD*#JCw7==A2$GtruJd@aS?Eb-@GdjXfRrt-pM72`WPd_4YC9aZ!r-@dN zjg<7o2@;Izs_oY;*@Fiej$Y?hD`hNYBes-QrT4B<+rr~={o!UC1;c|ictzhCz0&B`i@dr6J7@ zDC);*c*j=FTu?SJ1DB4j8*tVLlcpWGPLY}uLy4ijknw_UcRISyEfdi>k=2fsa zd!jD~^DQgzWp4^$afZuvS~ScBurs0_{BJ=vP%cg$0g((Fy`6u6v_98EWlGf|jnZ@%^N_=|weavazq z8Sg*r+f|}F%&XFaYqtJ^J3FB*bRTrO0r!S+_PJmc-b@#!y zUyByAP=GD1vZdJY&la&&V<+DH+W^ag7iY!oImHTAmaY(ff)wdF^|^SGaaP{owd#@P z*^HCVnn%^i5!9hna&7%YChK%~@DvcVQ{{HX7YsQO_OF~$(ua!tJd6Wf{1>bC!2)!^ zor^HPoSd`fXL_+is51-rA865BFqj#UgYp<@04ev}wjKfzdD|PxBA@1A2B)}P0$a}* zn1kBn!8@GeDs61=Th^a1vI>ujC|$`TPY)cg^&t&m@Qr(u3^vW zC4Gc9V6{4Sr~+iZLpuQ)fahIe7N$?!K=6Am3|TdJ=+xLxJ{;svfi!_lnfLJ@e1KO3 zo%gAW#CqO3gagh2i$GKouBV$U>9`9Dbo!xC1NYc1S{iz&`k}o{a#oqI1QWr3n0-{e zns{j5ONYUnas`_3A|KSB58N{V|k)#^PRkuCvslc=xIfiaOP8I)5 zD`wD249g;%1_>q^W3l~!Y8g6QbxOFzO{_QdkuzM~ChCp4i#3oN=t3BuFfI|0-;E-g z@{BNYB36895^5-kO}4slnB$Q9$LaV-ERaV6|JJstxbzooC_$v=GNe}q(aOozKb0@e zmDQy_YAY$KULm&iXMHq$(~|3+hIBh#GI};Jv6m-A^gL&DJ8VQ=bFf9ZnHgT=#V!RAzRB!@PK6;Fj6q%;u%&<d|n)lG47T-U#_n`m6((s8&~}l3-R3nRVePwGM>tCDvcVRXDr>B+xHVD^*n)W~do6T_ zd^A4KiTkkU@6J_Ay(W1kP}jFR6v7g!$5p_-0@j{~%@yhb7gcNXEAQv(IlM_( zB;r}+Gw?lCi!xF=RP-V@s5QV9K-D`6uhM}ryBW#%mpSbDqf`)HRnJvW`2~WnU5(}? zfqn^*s67j;XE9nIxfOtNL3$8ZDLcAaozGo3oo_$3waveGDo5LM;9?I;Lb6@3^c@BB zOV<|7&;f+ac@9p)+I0q-$t35D_AO-0cn8It}`f0R_ln%R}89#w5pj>aF`1?gKz zqvrqL{nVHH&UeutHzy+%59ZE<#R$r@4s9f;ofMEry^CqB6zZ<9q(E?k;B{}cn@4Va zNO$Ce4G#>}xi3X6P`zWBN_X?r#YUmocC}=Xk5BiMAQ%;R#g2H(yOowiUxqR25^eVht$4Z8$|FP{vm6XT z3~YA~f+157f(T_JT|iR(v{c$lgY?SO`n=PDMX80HS%aZN^Vnm?2n#N$ z&{kL*nUt?79mJNx{1jL(TdL`e^Cm_e;9sx=)8C%Dwl(#3cV`TR=6sXEib6FdWJ~|J z+A~{-<6iq#08OJak(6~_v;A@j;<-XQ2kTcSH?B6=kNBFh*$*k|oU0p?jcJ<^p7HGl z)y!Nxtt2PX1o~{$7WDkjE%k6@5?vr9+5OF13tMOhdM1K zaRCl4hLXEh33XYjtm2lwn{#B&D#VoUVK~J8kr<}|kK3SpBZ9z?oNJ}n=#2C|3C2VE zdYt3pP6n?uKCA3}y#fgkFcmA1LQ;9Jb1uaLh$8p*$arUDwK=$9Gp1!rjs5Y6{Eo$X z2h5_08My?!-bFjZ!}!Ahbjfo(S$+n<2Z!<>Nzj33ybuL^co~q|vKxM>+wG@y6Dq`` z$erojljDwJP~ay47l6|#hNREWVh2#bpxXI3EG5epf)ye(X`WK7BmAiS6N_+=KbCSj9t?^7;HL#g9h*su7 zvvbgy0)w&I`>wYji|>(m<9-$!|H|vxWl?{CKMJ`!YKMiQ7j&*#yZ3mF26gAv1qIu8 z@0ER*?A?$_Bgyl|4p>M57oe0oOiszaL!&JxCh@;)SXXhX|MX<_JNy^Cwatg2&%5rs zI0zuIR~+uTm;dFC=Fp-()M}^9yKy8H&l`CYF`|Io!-G|-g46l#Q?e=@7qvXy5V1s| zm=v~HF50^O3DN>pt#eLtPUd{37jgg3+aOH#G@w7wPSTsG14%x;CIzC>e zzfyrFV2&R~Z%g8QL7?|$5bXYRykcvvKV)b2i-gHG#8HA(@Ysy;LHlL?0uwDkmi0hB zp!Nx|;aM1&?Wc#?*lh!|1B@|^5?{on`3nTr5?rk&whDxO(OW_)?0mfO4YP5CBiQ3I z$Je1y=J#v3@MymWlK$tF-Or8}25lbWi&itQJFPDD$F`7Diy@jAgJ4oG8< ze;m(EOA%D2V}@V2Ptdw?F9j+VfNznn>Zw7ZzM#eF(9FFcKYI4-yQTj=i;wQf8zxL&yN`ZOs0F|mPcn!N+uffUx7ZYIQ>>TQGb`!+V=HD6PT5$W4*$Vm z-1QyyZCw6YpcB(>i0_!RP?%amOV>6+lt4h;!X5IEhVlHA38S_@0pQA#CNVe+*Gr$} zX*MbIHU(6t%N_9QG0?#}c#GY+$?xj&%Yvs=$rTfVCXH5^Yz}6V4oYgE-U0c&nWPFB z64GU%HKSG|WHdoTCI$Y>ZnPrG9Jk6T}cdJ zEmeRdl)6*%Tbv&k>LwSNjL6%*V>(93%k zlO~~`w*Y5wi(d=|aOTMRfTCZaQ@Y$Yp%eH}m*@I)-0a2pTJpDDjH>cNBnvYiQre(* zC~kJqhE_2V>ZAe0pq+|q`}wIvulfH_uC7O%y^E7RYT zA*w(xAoie;jgLc5n4vXe%W<=7I1#?48305G8lzbs`7LUK%K&)b@iI>c0zYb3HHKip z)lA6)#%1Bv(iy&>X!7VyHg-mwx^87gZgG}ELKtRk33Wr1}teaH>EE~sJ_q6 zlhygyK-U9d6Bv-gujfht&A4WKPD~5VoWMpXu_%RU)&mt7(G((Yb)g`_Md9KXfEVEb znZE_QX>h)s_~}v>20-{}cUxdC;v|vTwOaqZ!W5NXHfBkq2P+nu++Xk3f`k{(q_{G>)Vm=Vb^I#$j8uLbelUOP(_xz0_h# zimh(WhiL8G3Gak3ILD=N!z9hOxr5O#I=CHaxh~rAZR00j`WWAv{!c^FY=U1LD<`Ot+(xlHM{}dtK3F|iAY}z*j-6qg`D-sEr_z! zXusJ*gAcFRv4z|JVdB% zah%MePx`Mn0q&SNxlqg7{4ObOA7PcpjM2hi4b+74j9KM8cR&L{&|fAAuM5Sobwoji zqH62&$d7g)Fh1)1CKsI*(nB)|cY;=r?yS;9Ek>;XO4dzm5Az-1X)W zuOBIUc|WiUAQsk{b|#ljKylgYwR7CWYfPlIZGpy$5Qpx#nLOLccSPo2R9S2a6QoSI zn^?uS{M3k?Nn=_F9P{@Wm1XU|CT@NbQhY4=Z@+Uo;IyC(+wb(Ixb^6T>UE@h=uZ_P z0cTOe8o|K^g!lpxt7FH@>a+`IJKtzlwLH9;q-`{z)MCFlW#?^Uh}Da^qVXwviEiB8 zj%f3Z4#;NZ(L|K-1{pq|)H4v7m%Hqz**LM>#ezGmarr$HM>4&HkaF{rYHxSZbm5$&EgY% z3BmASWW;XuTnk=Lg9MW=4BiW#UIs$}CRCoz94XHTV5rdgZKK>Q9hi-H8XXfbtPSFt zB@Amw^s5d~3aDMG2ZJ6jXZ@A1v)nbDs-hN}gB~WSV*11JLRAB7oza;O^8EFNbj;&( z!-!Sv7u&$J8;YEPJN}Jn8>AG!Q1P$Cy-ZWU|$_HK%p)7QXTqDfDuN#mUt* z0A1)@793_R|3f~pw;}JzqErdrvAP}4$SAbADf z5Sz#m@ME!J6&?j?pEJVKvyNNIB*IjEMDUI1m7~ z`Sw(hNexmUqO;!#*C)f(5v0@7G%H_Wb3mCt&~H)~ zY8BJXO#@IzZ*9{VGvZRVsLEz7y-2$+nrIg-oU#-u*{Ktu6*%$&zQq2BcarBWEe8rf zp56%w7Ygla3iB^!M2=0#t*R3&^xk1)%Yj9JJM+%^JmE8@+(a0RIV{$KRw{0!CyfwW zEBPb76a!z34QuXF(P3CLNYaGVY6h1 z?#5f_hS)`~oCfTPOve}j%ul^Uqc;Sw{5sSyMd%5Tabt~r)?5#Ctw#BG7v3Vb74rBC zW3S$<+%nSyy0ZF9P(%DGx^#sf_!wTbCT9T3`!o*SsbD3+}w0N(V<$+@gwXEpjp z+7qNT1n}EN`LX05@V5{9Q*bI6)ol)9M%HlG>Hb8W+h%4~N?oAln~Tg*2?wJH2g+lVf}}+QRETH$v+ep^-A~*ScE) zC6LfB&8H-VJcaoDh}}rZB_&bJwLZ~W{3tQg?n4unpa^{5TcsvsQu?~S(v2!iCCeWF zhtO7{+KnO2t!k(nd|jfZqe!fW8^DwHEvp0 z_y+LvgJ)Vlf1Mg^d&mLS#=>I|^82ph#dDY9HMDqKl{l-5K{^>tyY%>Y6f+G=WAFyI z&tEQE)jbftsE6byC0D>H3R?kTBWJl{k+xXDZf(!nitU5%>V#_e0ZdH5Q^q_eG&YvRf@) zX7^87#^aH|-t70$p2o4j7-GPVc4(=S%y|UVgDn4!z2gJ|kPnU^<(B0I>-atqinZOw z{hQ=fLp99ou$phI3WusV{X~8#gk2)Qrc4&c^UVK9KjqiFI_Q@Nbxe!F_V0O|P;DvVNc$o0r;$P=rxRF8Q z#N4;3m5=wCDG3~D&k4g4SRN>R9Ohk()ujA8K?Rl4qeC~7(3^gA9%e<8Qrf*iRI`)B z^N&tzRrN>DFpejueXRPH_G=wvPH0ziah{*WFI7L-(vMIlHXZ`-yt5bOn4J`Q~wO$%U}&g zD8gx)^2M4c-N>Q0NaC&rHd`{xTeDX+N_Vp{^dOQ^Yi8UjNo z9qRDi;cNo3Vp?+iNa2NNBQSJo8{SV`hbxq1;}#Z1lv1sk4RbAh!Bh9B3Ic}I3D5=u8>D(T+D~UK?xog zHtEZPn2>&pe=pJ^Y3(l9vdPR#zpF|HrDLnHDbK4^`SSgr{N~m)2Wp_?oV$YM24`LS(#3ynIn)e}R2QwUxev+}^ zSHyS*Ka5@VLfD;~eVnGcW)k%0aBb{X{(F-Lq;8ZUBDhFn9S`$Vgz5WfPq6N_ z06Xr(K}0!gAzdIHaO2Yy^`@M6hHHEg157=^W`4(#S=W?_{s4Bd6=5Is~R zGc)K*%ESCMOJ)A0;zXh<66R4Pt{!wrp+cL#Xot7?NLp%_e{K{r?}0wZa;big+ajet zF`@b(zsjEgsY2t`(ls;l*g6IJlz#)uRoXu4;ra=Y@Ch&TmmV7Pf|P(AEEk7TMwooV z+-4z3^-x;A*U+}zNF@W|xm4Zw)MZu%pJY8R0*PNj({O$WH@7Xu9xPwhwMV13G**RI9G9356cKw~xNwN(HPa)bOdH={{9&_KfmJJ&a5o7Z7 z*h{7&a@M^={?0MprhXsFlTWgA!ge*Mp900z#EEom`;G)mA}=OE`v1S zK^3!8NCv#2Kn}mzJgH~{Ks6ML{dIvo(}${6^<&4p-%b?xI6yt}4;BI@4whR-rDb%0 zw~BE)CpM_UBr|@56Ny;yP)i4&tTZwHo0t!>j`94ZlS7@Z0G~nzwSu%i_SuEyVmD;S ztUA(`khZUG)<$Wv#h`b+5q;Oya;8GFTJ+QMZ7PmZ&6Sly$u`ole(0lQ6=T`*-YRtf z9LL-En{d*1*c4r^(^+pe{Ch4*(E~=w`ZM<-WqX^gMc=M?r!g(xUZa3>K#o`b1u&O& zKUh7xW&!*v1xN)UceK{y!JcSEN73Ny6v)&Sy_`6)+H!~AAC6`w*76{|WuC&h?djt| zP$wM=IYh*!cF$cbsiGFVJ)Xw1d<~v*4^cs@gF}o7xk^xt z0dk}Fni2g2^t9I&TAwK+EcxFV8DO{pXVLMvC`rl)R>y+w38F8Y0>H7gBySnRdY10g zJ7O_{o~85BTcGg=I5I1U?&V436KyF~0EWB;A({k!99Cy6<^*~;0bXh&m%aT#E6xk@ z5|ap;|9(9l!LykVZHZ@z44$@9Yeq`?fG{&|Kc>>v$9e-X5Yd9LIDJ;+L4bz=nIC@F zPopQGoIsb!ZnR3+4q}jRAR}Cn!tH!xOCMMy!k>=c8M?OzkBnib_|=cK!XY_vk%zx- zm5HQ9^|?4wvu^{g3UQU$doPZDVqEsI@&^XNO=eu9c(hTl5{oj@iaNps843KEK$4dw zYmp9z32@i2^tz?$jTyuL;lvD^1Xv2R0z9p_Ytjx7oBaNG-5|IxuAH=jgs4G?KnwbE z#)Phpp5tW4B|3W-@4-9zi91b}z>6%&j+9;IFduW-cnZ})66*-B^YGehE`t;%?*qj( zU2Yac;;7-vT0I#Gh<@LMf&n)f7)Ac!?AlC|*183Kw0AH!6}qYlNJ z*!TYUo?gIvZaHe(=d%|ruh{q)_;UX>Irw?4n#jAQ_uQPl_dQ{&Phb#8BLMZ>9G(ND z*-sClna_6!iKA!6W5^1MiGuBGG~Ton()|-#it4N|00XnlkmgiRVw#SZi-3?|Cr*}bA*ELFN zLBMrkmC`r3jMqw4%Wf)xBh70p8gA!1sfJWZf!H+CM z%n|ko%dw^soi4X4hMxs9v@WD#y3cf1%BYe8z}>Qp`@tFP1Y9N1fMwC&=oTdQ^yD`& zNi0`HN}{s)IxR(?#lGZHjBBz&u3bMgRylXA1LioC71Z^S$>octA|rlsqn-oreyGi2 zU=K-s7g0J-i($A7X10U`kg({r5d3aKS!FQ<;^jD*2N(7b>6`Tc!Er$K55)Ql4!oxh z=B(`})NwNX=4b_1R8LkK6V z5^4!aVAWWt5KkC#AgO8Fy!8b?zi_=MHy;y$O1qB@J)|Vg=hN-_X=OT#^^s@LdGUSO z(BQ*F`O5I$|H+HOwFSvnM(xUcwEPh6wN|R`MnX6z+^s5MJgL$4Fi{bsvZwOH`y+n2 zFe#G04k$y??8hhDteu8ii!V5bT{RkNHvxhgh^A8=l^MsI|Bp0lsZ4sF8YuIFn>Upo zFG+-(kZgr?J@__ba5+-xW=wacChoQhX@$n4zYIS+y?S`Vr;ylh)K+asODO>%FKFn_QYej5;C_bwd*Ut#<9S(odcPdVJX6^O zCWW!6L<62kbFhrkav=O{2tgZT=J|$SJjRvmw?TJt6r`XlT}?Ik5~CxUN-x@!&KP}j z4$NF>US2GDj*zbM)~Eug6vFbl`eUU6!sO*s9fl7*11RZc3rwRV;%2QnYDj)gfkzp; zHe$ckgAK?#CfjFz9m#FXG31)_1?=q1{oVrqL;cW$#)DaLOwGv^#5?55g5x%2tB%^6 zWH)kB)KXBfA1hJ=-!m)>=*8Ns!e|Oa;TVGt2Q^;n8OK8UD+ChdgHAFjk7X4Gl^MVc zYL>UPVV7C*!9{g*LJX_Sir1altf)3dz^IX9Pg(nowRX;PtP_peDZq@LTpoP9uBrxs zqX(car7`&sc(8EUTR~J86uX*Hv-Zb(MbB_C7NpvWP-I>>d+IppTFia-MDBJehN8v}Y)vGxtgmS2A(V%r*e(PEs&VLILuolJwDoDGtM{{7kNBbFP>h&PhdbR-D3P zvQ7z-A*GQh*TF?Mj}y*0k1gv5@~w~P_W5n6)$fe}3RoFi&Rv9=0!*BLwQnCh*=p9g z?fZQOw1DnKly!Z8ijPC?BhWK9O|z+@cqNR`#xuafXvY3dfwW(z*#Sjvh(F`MU8eCe zemCqk=#*sWtL&Qm&pR@a%!Q(ha`% zlPqAww2-4P5REpbklL(YLkIbCkspazWM~%rgWi;)BqU^4QsFTaj$)z9u)wBx$x#*! z{OS#`a_MSSD<6|64s2p~i7!!93yUd_5iWA`Nb{1RqzQZ8Pu@Pfli*(nzDBS*wcAK1-| zV@Vq(wgva*8PPnBlPqf!k{Z|x=;1|VW;R-|kadM3X)|3m-Nrm_K!Sxg`-C~F`;mp; zgpMAJpC5h?hJQbo8=p33lv^sdoVhiPWnFQ*QFiwT%lqu&3mr3XK{?CmhC)* zPi(>(iP-#6wn31=CcBe3{gb~hr<33RV9iZ5VP|DmU$=XJl||LAv4@hW#i*-)U8}6V z2Oze3Zh-=fD?`SGG}b(qugf?gg|H~6`P*TaNz*Vkk6&q{!{zIH=!W343RY?CA=gK0A?o`{Z=mfuPn2PLrGyLoQ6t*!rUGmI0}O5rQJPh`MN(3R)vR zJ5X3iDr|}1^zabKAEbQJo^mjdRULi*#Q*G)>%0neftk4v3+hPL ztQolHuBj}bW|nSmGbuUG!aJBPjbADWyo8*|m#0QdWV2=@xw-2;bNESMK7`JQx=bEw zV9r!+TzGSR!k^;M0OoJk;rq4ctp1Tn8Ha^*baeK05$8ADo4eMSok<9Z;mawybz@uh0JrE<@>`=O+>Prxs%sF}VrOQ?EvK4RqxSzBzdqGY2cjdK{f_=wCs3Q%-joS30WY@Z@B-7 zWTXvU)0w*UJ7%U=IR#QL@A2+NYS=M5cmwc14FCe|#dJW1BGW(9-2%%bz=o=(cGt<@ zxmt*9D1BVBVUP}0{VR@B6c+DTYYvSI7IiOJ= z$|cw1HIoeAzMdw18aOsZhkaW+-5;+>=5)|qv-@$un?CGw3;EC~q1MK_!5bg>;JXar z@4)E5O@|e4?o~Te){~V}Jfs7o1BGybHw!m{E4o)?KPxse)W4`2xrhu+d1M+7)uz*W zv}O`v{&yM`FbpLsgk23P-5;%aqk@M9&m(EVSKi7FL$fUy@*@>seibUZSfT=JNT|^a zaUqRs6Au~8K`me+1GR7*z6%)Ehj4PEgGUFiJGd6EooH9>-aiO<3iu`^I_lFPWNYP-q@g9rdy2L5F1bt0vn>pkW_QP*In-8 zf^^-ZKV<2W){`}p3jI7X#Js8JL%KnR@-P|X-7utc;C{=9%2Op7uv6xRAwg_FWH64` zlMOjAG8jeF^{O~sL|Uqz7&XH9u?yFpwH7EHJe`%p-bT3 zAIllmQ#F$a?Q1`c2v`USBG}kBwA1~mnl~b-Z#9%Nuzg%ODVGu1b+rsCHbA-#9@y{` z0%MggWAg*7oRPWWf-Vz5eJeD^K?WWX^Zo!b#ElHnyF@=D%=IqOe$$4-KWN>6VN>AJ z9-6UA>!F$%6zpp|iw*Nq+nf#ij&`~~RFgD+sr@w7U8NhdHCTNc>9D;S!|6~^mu?fO zB08{e6oagwz=wXleV`zjOog%q@&P^V5;OUbz=xil53Yv;J-~-e4;RyU(cvLJBt{41 z)(`Pvojc_K?s{K zAns#V6BQES7VfD6At;C|oVyE+r$wvT384THyxy=0;=*OUp}vm^x7lcq{aV4ET+(R=}Ab4ozm z7`oQifD(hLCq~*mM$~wnM6hdKctHgvXe3)WfrtG_s8x>Qlu(5__@U9ge-+YilGOj< zv8&!miENLRsMn)5lMwl9K8q41xT@Z|-Jrz%q;fbVc)M^rzXpTI$q1|<10%W!;iCG% z4p!HLA6vlQG7_R?Cq%j4M^LzrfNC8PqLUD4VPP+K;xjHn6t=Lq6bXbFiV^wdB;IfC zS>$&|#YTz#juJze<$SLnx^S*XZYCwtiIga=tDixAf8o48bmIaTb`6LtDS@8Uqy+Q_ zpoD?|Z~x@PP>|>>MK%nt>f9JKr{!^WC6+AQN-$rjYr(M_R2y}bKmj{@F-n!{KD_=g zhEw7sd*M}-*dH61L*>fY$1;iaAdhSxE=jNs$i>5mZbD%iu$<&WGwDf^>*s zix7X-1}d-oBdG_3e`I^YVqimdfFJPU1+1>fhG38NxYuJhGdP5=@hmpf>?Op+K1_|(is(0`~bilHaFnA>(9Wvg%7&q5I_^@83l{M&|k-d#Y zfn3QtGC+squ7Onox!#-e!NbF!-+PxIbDUo6C4eCsbiVbm1Y$jP^EJNy{2ET9gU@GL zyV2slKXmiP2Gt={hUlz9Wu|i=LkbGjS!7@v3!9Fk$iSiQsF`nYA*8E(vPNAol6SHA zG!_o!VIy9mBcI4%MP$%f0e4@KA)n?h+qijWiw?zp1&6m^L*_{`^jaSy!#bM`uRmXZ zo<#;reK;F#bhz)&+`O?tXNB8$b>@fI$(%`t?Abm3cwns=Ds|Yd?q>x(XpZosY%^M+1gvf3YO+H}v9Qoi|tZ{UJ50!MN+?Ctl ztVlX^ZrFrKONR9CGY1Y^6pU7g4(M`o6JlV=I^>NERcxrvhU#>vpDCWj;%7z&dn`Dt z=WZqwzQ2AyiwpBM?_99`NpXMdCi~cp%!X@H9@%Nwm}uApE^L-7L1#teUU69SB$0uq zI_?`zhL*1LhN7!x&=A(3IIdKZ0pt8dQhKnViwqJxxR5%x_*>f-AHHT{;je9?&WA$7 zGLM{O`1bnkG&-0o>uq0n$l>ucQrFDn=J6&vR48I59V)LGkVujai1!g`Bh7|epee`v z>uwqdY2}qTuCk<&nTCN5tX6aN9Y%gpwrs=Mpx9u3Qe=+}Hh!B0&>vb+phd6Hbe4z0 zLtYQwOg?-)j}JYJjSu@icfLP(v&Dz1dxf7AC>_V7Y3^)VEfRG$fDbfF^K0iM0XL)K z@8{}-sQG$TYNd4q1=wN)a|dJL0V?GgQ-)JwL{UQZYV17`CCZ=OS@CBqMNN%ZlPksj z0Fl?DH#0oEzJ5K85Y?Afn1!?_#QkK_>ki)H%+3gm+Tk58Bt%mch-z67f+i?>=12(2 z+YnGZmn_%r^p&d$G8-MBLt=z5o)?+tMf3jVFtNeUi(<}(lMW1CEn?x-!g!yUv>Mca zz5g|@+7j|T)bP&h(VH0^#_MSUxe;7jKOdYu9qy0bywO2-3-^2MxcEh!43&K7G^@wP zQu1NO9r7V_o6+P$uF>-i@ge6cF-+yQf((}_@qz6uqW~iF#4uH=O&YoGgL%<4a=HF# zH=sY~7-!6Dq2UT_H}8KKK&_4C4oa(P>A zW53yV_))2xF<3#-mmgYZlGo!mlMns%G=g>AEY)oQVrryRQZ_xWb)yc%QIoQ3)md z@JI(~A7a~rLppm^Wk*tuhx%I2qC<4|VcVO)y!MIyK(Si06;bdOV;LO;NUh`>eu;^m@c|G5=p*$QNbk+3YB&}ajjfMsy zLt@UWCVUG?2-Iuqe6zCnkfTO5B6p)<*|Rnmyt`{!(ZwS8abav^aWx){e?W-fxnxnJ z+b8s*JJ#o9f05c=M?}1QK}vK|BFBeRa(O+CGs8oAO=nReLuDpPm}ZltHw{Y7hr}K?#m1Rb`jpM9DG(;0|7JqFiq7nrFgL$BY z8W=vN=e6pTf0JWn(?U@!|eJ zPIkS$>8i;tL&tk09i%BnbdGpqIG=>DC_-q1MN6u7W8^nDA#%Mk|1c;pllCWp5cGSf z+>pmN%P*KDgfl{f2wRBwvqHH)cVuAEJOvsL8C*AMJBj!OX>HI4}AyE;Lf1c`K_IQekGLpaFIL zpn>`pKBt$wOnq}R-fvRaw+9MP@NI-NBlO33u+Agn_&I)_#e)J97x7?UbGSc*^F{+* zGx_LtYQiX(XW{_q7@x`FYDvQ^H{O-u_lh=jN??$JOo-Pz42(`>34?lP#~W?#2c~a)Mz4A1;Av#lh*d5M5Sil?;tuG* z9#+#h4(Jf8=uo(MdC~#)!j}PpYm^d89pXbH9}4-vGo!dgwRLs6gOm>`ZxJG%U8ugp zhs^HlXb&A2>*<>r9KMckXAxpsdt>oNm-~U?n~e{;XExD6Q`+2!o83Y(swDw7&{hxm zrpS<5ar0u24b85~k}c{^;Z%scQp|`GxJHT13~y+5h6YamkO~sPq|rWR4D{%tJWn7J zeX9aSDdrbr%240`Qa%M}w2jBIhxPEyj18~j>uF@LO<+IYpdA|?7`|~fI2p3o!1>HD zpO;EHR0-6$cXiT1eLcjCDn8^PKCorjo#{ls-iBm|R(%zU!*a*?_dLK?Z| z!+FUya_ML|Zyg)#H`bvf_RX7-+#N!!$8Si8*Ru%GqA#|vaHGNf!2Hcd2wgCmCl8P? z4Au`p2;)S>m!g_Oui`_Ceptv90)m_TEwFqcpbxK}3=Y>HB9v3l5L5m&jq=OU> zsb;4`{j;~tKUv zj&&nSk_&@9tl?V_CK;NqOUl8?Fi*}aG888RdRVy0vOZlh$bcJtrYuop7$SqGk)AsA zc*91nytjiXmv%7s+&?}nu&;S*wLQdzbsjlJJCO^ed&45!=x~pR6UnZN1q#q$13U(Y z6m63ZLp{yhboM@5Kmdao5A9Dy zA7i6~GKc0x2lLFKmEYk zyrGK)9jYfKDqUq0?I!2{O4^>eUKJ;OB_%l63rc8BQsZ8dNr}ka$yb!%K@{2T0wpSY zSo6ySz8~mrxxpYVA(cu@r-Vc>$=YvHMBOV>FOCa~B!snx#<9wJ{$>VIg_LsyI8>4T9E82x+Ur~|JS8Kng90nbQ+Iwgq@nIWXj-*7(k>tzPq znUi!XfetJ@7>|p|4D>VHT@WVNWP_gzKU_kMImrOaSdf8}-cdWUz=kX~^x^|Hh8t7fgg$+6u2h+{dr^j=JVEOl59-ORu+(LId~=DFm4 z2U58|aLMS|r^S6z9&6IOJs;C9u_QzW9?%&F}L<48pYcN7%0F1ZkO&x9VD zTwvLuvFuP78j3p#Ho2C!D#Zihy~vQ(Q+a=Y3?H7!vq7;I*2h0^%Pi}un;96wd1Od1 z)Dj!^-R+Vd&m(!Y(8U5;KrFZ#x@0y#C!(_f3n`>r5F0oX5jCn$Hk9jK20e)lsVFi? z6W0_P64*f9JlQ}sRP%|v96U7KoXGo2j|%kQP&$1qHms*^M(jUFIE@a~S6fVU_eF>M zb2m;09uyWNhe{Bsa;BDa=tY+b^rt5sQYIS)kA>#;E$D8_NpP~^s(ID3ZWPhf&FoAX zc%jHVCXyQoHdAEqdAuAwGTdC3_eZMJ_DA*^puSEfgxVgX!a9Qt{YU?K78P>v#TIqO z);$&OXOJ@du#iOtWsnIEYNtF9WEh%;OG8D61fLEZp)`YP6#$PwaKE18WP=+CMw*LV z#s(Fj&?E&L=m1kYOAeU_MQj@#JTye7!xkF;Qb(X0bU=L?4?ua04$B;}|LouUkJIQ- ztT`R_%D8fW;^vJGMRf3^;@zs$5;%DaI&>8r(hwa?t!?-KDmawtbV%0=4LV1=0!x?V z17!e>Kytq?UEzU8MHtS9M%hctsyAd8#--eTvw7Ty8y5J^r~rScw_Vm#H)O>3(-<*) zM6=FRe$R;enPj6@Ei^#Lo=LAx7jaNc>s7L%buEJU%meBpq-B2VZfT;5Ixm%LEaFD6NYR#;PQ& z%ai!fueTX=?sX1$yu}CfxaVbteBfKtq=rIJCLKID6mMds{J|FYZ{3`S1COQjc^P|* z4(p8aYae}_Mu+Co*;;QjxbF|&Y;<5tasH2Il!J7ropBA;Z15o!@`0L%$Y+(S8%YSYuzZ6a`XnSo{9W?6{>PNm*uKCb$zxsa z_3X`z4*fhrSoF;nwchA*-ygmy)d`^+r>YMy2!w}sB^^xrZs|o0I>6KkW+*Zvu3COL z=}@!N!QypCa|F?$sD%|1n%p2E=)m{U+?=|*JT9t-l{a}@EWcGA_wnOmkPaz7R*PB> z-b^~oBk?pkpwlKg&9S0B9i5aW$1_0evu-`w0DkyMEJvF7%A z>Si7k*4tTBXj_rG-so-LAG#^#OwhF=DRIb|Oi0KS3+@7}5!3AfRLG)2QF9x+lNA~? z%RR|ZuJ;(QfIZ)hEFE>EI1a%8WN3J6ia8e)7LjGO*bRG9=U`C)8MP zMKZvN;gSqVWJu~@!4(7}gghn!-^Q$@p=+^b3!X*#og(Dlx{LdR%a4qVaiQlU{j2rF z%{(G%ucvWgxNChabdA^^E+rHUx7>L94^u|`fB5yxpBT0b1wVeC$8>-@#}kG#=M(J1 z+CwQ~4#W>KNK9VNm)wPde_rRmi{^Ka?K57y_y>KT<~uguWPSC_H+ueaY}7+1;Q3`< zP5ifM5{2nF)F)7$sJOZ?SK!pN;ceuire}CgJkgPA@{FG7_#Kc=hW9~!Bbei9{KCt( zahdAH5PpVpO37vz{j>QtB2UlzF7)lenE!)6$=~Ml!o2$>c_TS;-v%vIG6k;Oahx;P z8ovl-%)=l_JAh(66LKCDnLE8F6dc7!d4CBy9ty$#r~Vd`8Cb*k#I-4Cm{vL0M2?tZ zFD#5fQ5c_3033ADaa9YJ?Rzv`k|G!kYB(YK3~$}57yOD9O_WJO7+av3GrxM|Y7dcd z<0Ng4>a&XbPk2X0WR;$){~G6z;rXwTl=KX5=E%u4!Hf? zJ!%nE;(VU5g#yQHINGC7iu1j2Z0s5CGxM_qP5|KA+pb^z^AmO{2i{N87Br1VA6yC2 z_M*Y8V*cq1q8qv}sK!x6jmkYpFdy!OFa7o-U<*fo%A6G854rsHi!cxb%Llf9LhNz} zq36^whrRcZski4ew%8T6;19;r(7O9})+Y0hCPglFb8ml_Nb12UD>&+URN=(<)$x{l zk=NRu>NVUwkqc849F7Bao#aU#j3u%9D~>yIMG&~z7+Ls`L?qhUD1!1X>OGkealSXt zFkpc&4f7r-#RN@96}6Y=0n0Jk#-|B&!k4Y{9Hdz>uE=EpIz4qtnUfR>c*48MHxtu*n)OTb=A z;onW#2{ylCULw7g<)4f+Hn9Wg(Apm9JF zb^>ccorj=?Ua=G=tK0@psOy^8Yt8o0Z}>P3YAaDt&l}Pn>T(b2#%1nCn^E2gOi5cu zoB-HHmZ==h2VY=QMLj5^l7ZQ9u$j<}Vei1ssDN=eu(9aQT{=X2h7F)i&oZDL7)D~} z8=Oj7!@WKqzvAO8xEq{r2=@#md${{OxJ}l6vV_FlM7t)mT>$Ho5p9EWnI@ouyQ?*g zrUF-FDhWY}>xFi~ztE1HrO6&gf{w^cw4+W)x=)uv>oDxC4vN~TEzw?!j!#CrbAdUQ zZ|!{$YQB3$nk>1$1SJ%-R0G(8T1ZDHHfwY;!voSZ!3eCa3F$_p>2DBW&0D{)ADo+x z-8u+#64F7(BWJ4%oWSSX0JXZ1j_Yh>t?B)V{lBkgA>H92MWmzMK>F_aSWPiH@=~TA zD~g7&BE?-aDH(}!M$fV@R2>FrGu~sXx`SN_cGUS;2sSHk*%Y%O@yAz*3?oS~8}CZN z4(JZBX{jA33|)e~J|1B$I?e)n9*UV@&l}Mm>~i;bEPlh`lr1Dp&C&KClA6+ypl#e) zbJ+^%K)Wc<=0;Y+odI_rIv|<*1-b}k&vU9Le_kt`(Fl0c3j%;6q#JhU5%9jgdaebj zKRZ571Kijbf?6-^Z{p?dAqg*74lH^g8QTrSGZN0W1$Al&xN`08`5!R_;Mwy(Ms2(u zz}tv7yF|Gt*M=E%@*yf{-%WV)Zl5Vf!`{EnXDY7wy%u}^Z1^|_Zn)SdrGNA8djsyf zhomprmyk@j zX*LPCwE!tcg_GeaU~CRIBnyO_(Hl|W7v9{im;E@diHBvyKg>+?h<75~nD|2_FbVFm zzTK?ViWBVmvz`Svrlg7YOnPtNefO+Paf6`|@7(=R8%j*LG$6oo(ec({Ihte!Z05iq zc6j)1EC;r()3T}l*NC@1Z#2P8wEeI|EuL_v2)J9}k%79`67KbBS$~%E;7%dwhBKkv z!)h;HVWb~wTs5sk!Mf9%erQH*IN(mf;qD&1ao8fWQC&)X3VQYD=&4}f zX<3zZr>N7Ci%JWGhAh^zhZwlGcm?=cZ8zbbAEyDXr77?SOl=SN-P1C3@y<}M#tNzK zd`GKCyS)iGHzG{H1-GYygb8przrBN}NNwsnxRrT(!3)FR$(OT$EjlcDk1edlJnsW9Iyy*_aWk2V%-kFGD8S8@Y~heB5~RepIM_NDTuZWkMtq!wb}8Kc4A|2RwCx_+z4@ITej`Z3 zJVh=ZOqxosZ(yfqc~~kHl(B$o{|Pp)I>xDBfo&aC?&M#lRGpQHsUTz{=D!Orc8{@H{&tt(S&1<(Zg$9YhjENXBsOE;kI_np4uF@zTXe;$obvsoNTW)bf!~Vd!Z`791 zp|UWAX9AI#U~f527-BDTznCJ?R-X)_5E36UCxE?9{nqNAAK&qD7T9e9O|avQo6Wd; zHtHtRX?K;tWG@RciZt4FQ|Yjw9fh_hepw+qw+(C_jdK5wMwL$C(U`p&GgxyiU3s5F zjX=f$r})HAk^psDXMJli&W~^SI1B2IJ&J@nrCqz-!rjBM2x^W7plKrG0bZwxbrZ2u z_l@B?AJcHGy`7M^B%3*34e1@$ndf8LN?$wyIh_Paw$4ZO$>^E-L@P=XJ1_6-w$}Rm z_==Cy&^FKd!X(BX?Yrk=VavDA|E+|Z3oqy&XoNfJd`zmhCKsJ>f(K}OJ0Wi=cHt3O znvO^+r1Ibq>Eedm7`2Pk4Wa9ZG|mDgo(<5l43hf*Z972w^~dX3XpcB_qnT7ld$jK! zkvS`97tSuUZD3w2iGg-kg(NnITS<2I*D?I)1^;^_I!y4cU}*NZtU= z3E<|Mk^*l(pk1K#;w^Or_gc9#!5%-xX>d1s?#|PEq&?j6?iraFJ37xbz+Gj!F7teB zqL06JX7{b`scr zQ^jZ%(9X=YLpH&k6t&?=3$6cr$LGHfYi_TlI}_~raTeUTz7pRM9WZBZcp_C+T$#$l~As5x5Soe!`ZH~=vr%{Jw61%0@_BL7K z;2WGr2H{R=4fpz#G<=)|_sk)>;GXB%9`55&l7V1PxT*L}n&_o)3U$5QG)xlYn~m{s z<#DKFyi_?au*Nbo67xL^P&h!_{DyYH85#XwirP_Oo1`Vcoba6UIqB_vRQ$jQS+YEYMCB+d*ZltG~(=Oyjx`CsuKLaH9M zGp{yC#sg(PP@3Ty8MncfuB#1NKSDa0Vm2=DG)N%QHLj5^2S}G6J*4|*IXlEQ@?5 z$E12!EOz~Ib?IMG{7BiObz)l#9)-Y~Rlh(Fg6Klaq~b@WuR5kSiM9%a@Z+DkNJJ_N zwAWo4TT6B(*6Bw&3+)2K3U0@)H^46Uz~+5%Fg610Rurte2N1I;u$|YVq)Egs(RgHNK=8edW(ke0qZO^TJb@I6ud=~c%PY}<{j3VCu7=VZ!G%p zma0`nKs!)rf`v2jiB>B>JFP3MwpQy*tmnsRXrpWcv~Rf^S-p2QRt5$XrV(_99B4S3 zpgcr{D6CE(+Fs5!&-zN!T|ExJJtkggaMKTRzRH}sEOa|lg^O1j3ctC^U1;M3#zYpn zlUPsutnv0G*z4ml;NvW?Et}#kImKbl#WC$P-Iq}>E&586&V#_0CwewHY^tY_7qtyEb_%qGPDbP2$o5^;L7n_hRfH--x5sFn^F`hsV|)j- z{QX>k4<8Lc4dD|Ot8;2z!3$WYCwT`0>b2z0)*2l)4t|^kb(0-8ZoB;k)c4NE+;l$H zAgpWHh^y8C>`#HUH^ritvzzihKOMPinozSPhN|fs)=g`tU0E0vu(J|BisPgA#3E(L zFkua-t|nVwX4V3oiS+#MEUY1kC)T-_zQ_9B@yI^QP+~WsHcK9n6+%W=2=UpmsCeDn zjIes+AQq3;k*6b)mu2i%MffDmS7l`V8f9RU}L8zEF%)Z?&})VTAeeI z{`URbX<&OQlFQ!E$=bccF%^Mrl(r9ddc`hol>4xjP%$K{0u{ZI-9vk4?iN&uCE5g= z`;Q%LjKvps_i2na?B@z>g=;(kfEqw!=M73)YgVs?ITPlu=K(#k={l3f-Yd!4y^~Qz zA*>TkM4GZO+*cSu;J&YNh&RMk8iXOKhmfMHV~ifI1Mf2-=-X31Urz%YTkZ&UEqk#0y^}E#gyA_J_$jOzLNM8` za)`IXq8G8d=X|`x1M4lr#%D9Gxdhg1d1=dvItEb^h8$!rQLC`d%0jS2A#uzDSf^!E z-PYQiiS+z92kVN9fIcQk5(eUM|oIyJwDqstDky{sCY*s=OGUMXMojzRDU z66nq_YWamVjE4(1ViN7lt7DQyPcJ$gkw})vwVE&!3j}+ug}1dj=lj6NSzyO#s$t)- zFR?q~oK7`J#IA=(sEZ4oipP-TB~@>PY5LK)te8-DosH;Q3F}O(6SX;wy~Qe*PM0u5 zB_h=Hzy(NWBgK9I8y=5?ep`VJ%N5vbZO(+cfA6P(4K)qHzHv>zqs^IMqo)osMFkO# zU$7ydD3ZOX>aDOCq7MeU@#)anh{-UbO{}G0&>#o;>l16Ajp8hC(JUHj?t@sX7Z%y9 z;W>@EYF?YS*jk!1k#67HSy-oBmGsR!z+S;{N1M|%!49yoWEbXO*&By`7vYR> z1MG4KY#Yt}E78@Z$)L+YhQPKYu$|i7J7HPbi21xhNbaQpu^y_0<&D~?_UYd!gfLpS zu9Gpdvk*80!jX8u3^Vs7%?s56WU|0*FoWzY3*eYu zb9$}GnNZJ<)4;|68_^DJk9NFwHgbF&XbWs9njqgxLfaHSx`XYIdCF&A4+%kcbou~u z=QY7jn&E8#yKjzYtf^8+h1GOA8>xqY6*)X*(X$}fC9lC=pN$zGXMtUFNd$Y|F!CPk zduL;D+x$`$K#I3-J;65BREsylHl<~Buu+E|U)*j4_M?Ehu%g;E0hVBt$thG1JIQsG zi8E4wI*QhH(BBYmFTS0&M!Fv$oxZ2jkRG(-g57G|Bc1M@izO)w(cL9t217`D6tPwX zwVDON#oDo`4aPqnN=gVe&&8I0VLkr`SeuKgl8!~XU2|lQSO*1cTu-5(CDv`J4{2*T z&O|zXk7r?RHD_WCqq;rT_l`wobXX2Tq*d*R9zUAOshkia&iSoWN1OpHN+=$zJMS?j zxk4|R+K2$!^$MEVNL-7{EhE5L4KVN+FsF#pby-_+wY3gsq8z@5vv6*aR$LJVPGyht zy@Qd{>cF`vdT>Y}x(hq7Rz06pf77;a0v%|=+&Nx= zdG;mJ%Oi2({Np^NyT^_hVe1~~dnY1C)UkC8W<)vp~il=^R8ZizwuP=4Pjn)v`Faz;2?2ySX1kwnBR?#FH51y+GZACZ zt`aOb+ND2vIf`X(osSYMz#$Z9vjvuQ%tkgEfOcQs(AP?wiS_HZ*R#-$IqE7?a%g+B z?H<~`+Z!${d5aUY2@=>cLoM!`7+gi0>&7=3b-?*kk{ zi8Vq$7M0If#vVPe#x%nc>-7ow`fYq0r(r#k^1eKO(jC@!gg8w@olqK)$yxna9=wrY zCoiY61%@#wcdGCTAEPa##f~*&w%^c|MH|k??xITt>?EWqbPsHQWqzh7_LMO(V{zF; zZ)+LOgt>p~X9106lEk^>zQ?)T!I`7!*z>|s^z8mvFv(T~NnS~1|0}(f>Wb-b#1=W= zoc(%kjMsP|IC`=nw^TQuu%kCfXC6D6qV~ov%eW6X*7=orQFcN#)cW z!VReJ9gSKmK7$Wx{8;(X6+cjR#?Mb5*5Gqr5T`q(eY02O1--kWT(7*!6p0mdU+K7? z^DT(#(*WwI6&E@eHOtGM*OP|c0d-ktA!`ZF1lmr7x_QQz>b|Ml+P!m;Q|W-ZDS9w! z;xg*d13N+pbrRHVb3+rsl2-nmg(@k03?0)Xc_L(xPk{9;)!v zd6j8yB!}Gxz`EKStx+I_%ezBeS>FpoP;*}hiXcMz36E3Lx|ah|ua89gR=?HLa2`>i zI@;Ld-0vNU97u<(92y%esLAh4#{sm&sd+f5-+XG#fgVJ9ORXsi$@v$jYvqyXGi4h7 z554G*bctc(i|?sD(Oe$wfpzD+t@Sq(=kl$bh4kd261^MlajoAw6FG~{&ABL1gKXKo z^oes-B{g>^wEM97HqLG$w3{~f$`cWmWK_Hp%}Ahqrn<|CGDxHVA5b7o4;VzM`KF)< z&ryQ59{g`>_00r2f6HeX)?)T0_&Y^Q_HIYtk%1n=^?D85nV*%7&v^LN3 zSoj!H!YAJZ0qbF%iM)QJ$`9GKOaCGd#UP17Riek97gC)Lq%)fX2fcx{@@B%EzNOQE z)}GM`^uCNb?j4AnLC4BGRFpCX$#g9@j~ymasxF3@bZ>9#O|({&HIU|R%`6--^CZx` zdKYwIy33<_Q1p;E0RpJMZBBm7-5Zd`OI+30cbB#BW&$0*#j~J>Me~F@P&2s)``&>V z**C|~RVCP3K!Z-2uvR5Cd-(8*wGAoC`e5?W+%1lq@{AKOZ%`xh0PAh&Mkiub5Dz*L zStrM(bUrE&e{#sKm363I%Wfvp;Y6&5kgogPdOh4b5*kfcp$0+QLN%e* zrk}x+5B(KnkK1$;Z;Bx+ig%rgYM+T8n2>04Gq_3qn6YL*ff1C!l(J>BtviGZ%`W@+wX#yQf=Yuwc zaY7kzdm2L7D&Q8{UbF6=@v-DR=$R-$~doqh5$c zTS)*r85#tA?HSV7(whnO_pc{)V`%P?33i)Q*n@3%z-}zSC7FKY`sOWOJd6yLQDFii zw0${^8oKm89C8vS*t{>o*!GRum~@NZVHc9y)lvmjuP~>Wk_Gk zZzk5?zJ5CmZAiI@cD6m*cjPyQvyNypR7|wBf2MLzAsQr__|I&8VJ@!t<|XBQLk$9} zE_KSh%mnTF8BFeC<_TH0O>nf}(;va~lVG!03$4$5n_ynWQG%YAH}t;N-%P0Ie`kS> zt;_`5#(pbGPxnqpQy5{pC0EXW6KjMKfUSxDoW*PnsW+9h0d`|LV(u8nDwJt+=>|4< zo;lnFtV0vn#5y_DrV|oHg2=f%K)p5%>1zqj#QF6+s8bNFZKhFsr0*S&+-d~-nV=w| z^)prWduVA#1(DPpYF|vaWqxW8@E)1`Dvs7?e+PAC%+5ZOO7}ela};QaH4CG|I2$N_YCSQU9*YoxYv?;nLRUQLY(*xf7FgDu=3BmN;1 z5n|1KY!E&m3IkwnYQz3wpGs{qO|_z%Wf9sXDf26VVFWsdaJ134uZ1|1*!^oi4es8R z`PF%2+reE97aYIP(2#I*gdc0Gvf@bcI6|Xq&2zrg>^4FjbQQ|66f zk9N6-cGL~BjqFR5l=?+XC_!SE6i6@%4YY?WxDcfQps52kh&43`D2&j)Or{vrjb!O7 zxd2!PBp<0S*#k*UirRRhxVIhcf|1tLUMq1X)b`cR0=tIR1$It1z`l1xCRQnu50RQX zHKXiS1-92#y$9CiuF${XMVZw7b^97?}+4NKAvuw=erCidK4)VsixuCNENRGs0^%) zQKb6n!5+EV5wdA49wOEO)m8IpN#u5${}z4WgtcYG>=|IN+t~>9dO<{At8gaN6|~bp_C(`7^hHuHFa~Sw=MT3s5Mhi-7k*K0jxJ2gt`*y2zlA&M^a@4JfsiF^mpf$ zW!%-V6(j?0%wPwu&s)L0mf}oar?2@exLaw;{Zbt_;6A=SR*s@4_+`oQx+q@5$zFp( z+O_SyuwXVwLY=a__%c&t>||`jnnOy+{+!rZI}Yengv%w7 z4j!%p?=!8ocTi{C)2(d&iRCGII+$qmNvQd_0P2}cVheVCj=T!3kX~zXCeHC|JPm1U zO8R1pJJ~|Nqs6Hb6eP@gx=Afhzd}C>&ULD|n$!;KZoA3>4q%-KHtiS)HZL@dTYVht zI(Rr{H()iEpt@x6@cR_l2wkwIn^_BX1vh}bb`I%lEzX2Gd<|!TJup5`u;;0=2ixv| z-AsogjoPzo&H%O&k*3;e@`hNC@0NE2tw0`Zwg_t;jV^c^)$^%3+u>8udaI4@HuapY{c zDyzvnTx*#`@{}(Y)`DUGu3UgO7X_B;CsaQgS!sny?u_4D z>q)TJz9D_B$eC#WI16oTnW{?qLfJQH`n{8p1N&HikrYF;W(F-t#gI<*LaI*dP=DRh zlP@G4zFQVx^K3M#7aVN75ff~eFJJ-9rm;-4IhN(qXyBl?}F_<{aiE|1}6{cR8vt2s*UcqbdMtH85DPmqw8Cx z`!z2FxGc6O|JIzm|ehIQt6bae#-%w9IB zL!#BEu%_6(@!0TvWf7|w)>to($I0ujuiwtXI;HAh$8-bid&eWqXaH1^DGPIGI9Vc$ zAR$mH;@z<3H4zH7lj|g5?XIE-Q|0-%G!|D`Tg?ScgX$WFqL3{KL!#y6OXO6va!z~V z3T5PdF|KbjYh})4_1D+0r(q3wGqCQqU!XL?y#rFi=+xfHz=Gp4MWt4Nt}qh}wEaZd&4V;Dp5BkO!bU$W9R3ma09s2Dk`_6*CaL-mjb zG535)&vEqF_!mu$`>4+vv|$sq+~~cq#u0e<{ADEAaPeD0y*?XXU&re>3+e*XJ3!rf zx&igQ!%_3-Y?x>)Iw@Qmj6~>`C#0R!=B(~2KBdG=B)bdM-;`2y zHU_4B(fn*wGYsN@_B3e85V;t31@>B(Gr{h!{WP#q@Fm!JuMQsZ-r1;`^g^dL$-b2K z88eZ{ol!4tkx#T$f2yAMCFa=K%)#d2SlTbFV?!QNUAuscg@>qg4b{&^wZa>v~K5NJ#%9veKzV1tnVF-Qu%~ z_9C@9v2HvX3$+Uy*qIUA19leFM7sFNSm@h~0qUp04efZG4oJN|7yGNdw$p&_RXErE zwv&zwdH~5@Qw^$zNLbU=dk4_+9?HIX7jy*BQRgDSopBlM^t!qI zMc!qad={cZU2!6C%db5#qnTKzwR^}|3vwo<>#LoGwUy#vhkgU>#t}F59U^~ zF6q?+NcNg4dSJEi)Z%`npcP4`s|*X+R&}$1jJxWntVM1jMSoexqkmo=jg3!>TrROE zoW#ba$f!LYE3nr`qfM~uSztFb{Q!2gQukmVAB{y?1lfLTcw?y0hY@F*QA@PS{h464 zgKb$I$E0pSjvlTX!*#ayj@cZLu5oi#R1lGLVcvN~ZNs6E<;p)ef)JVctk7PIawgX0 zwVZ}F2t%|d7j|gh5#8B)&x^thtnCig z4TXHJ!c`dIj0UjwFj7@kjXn|9QvQ`;!_kk|M)z}X#!_F7?aCaW^O>@kG( zY;B7o(ov^li%LUs;^~;&EzYq{U8f^6n9eEb;wB4_X5uf(WFeSW`aEDmUD&z*Ai{%t z3HDl!Gr^AMfjvrA%9m$;w+Fl213M@cu}$n06tJ_05EItk1Zz<-qpD1cDbpiHTi}M=C&)0Ao*y!aE>`?Y#*Lx?U=X}i+LTXUXmpp_t z6;z}5z_yG}?Z&KD0V{w!?|6?%F}{PkqK#f$vXFIWlwu)dPp|3R2RY#k-r9;9nb<6m zUZ0C${1`vaLOLeZLXw3WK;JtS3$+7<>&|2vs|p{wQ(S{asVYqoq5X0-U-x{EdyVIp zPTm5mjMb5TI1;%w8pf5XYAU>f@%MrvNMemI3Fi4njb0=$GFe~E*D{>(bHe=NG?*bU zC(g0$8ZyJ~9g2*!smSk*w$qB!$)iVCB{ljqm?U|fdtT`$Oqkt;nH^xx-oC+X9FZ=* zo-hrbh+Zb#K64w9#xlvbnYbd#%EmwEjB2 zo(8r>#p=-ayOKt`cPPr-s%oMX^}MFUnzBf&zRe2O4z?+;9z_{Qk@-+`qc|3&DrOU& z-X;<%jpB-X#B0sGJ`JrtiM3i^m7DHZyav|moB3LUGg>l~!mJkkn*Kx+n!i`0N~ z)dH&!lz*@mIKFJ>V&S_VEwr%5>B z?5y^kY{7HABJZIXJ=e=zHb3e+oH-et-C{v7Cn{D)^aD|Td@wATD8rc&O0@Nbe66~f zD9^{@43vjyn_V55-oSg156(l5vGiw*f*Umdq?cAC-i;z`hOJoma3TlV1!0-U|tMrgF;Mn04+F__NsczaG032KJuD=Xrxg?qoagQuw3b<@ey_6?pA& z@bUq8YjMko;1w7_%`Dpr{4D!EF*tIYA$XLLz+DYbZQ4Jz`RJ)Fzw4=8i&f48FBd4# zX1Z)x-DW#O%=umQJIu2`_t>tOXYax5cfnh0PtF5xUO`c(zzL24;hi30epl%s=JgBa zjX%%4@-gP+PdG0kHFLF1@aUkB_uxHtV6;0i?}B;pqd~9Tjd{6$26(tm1H6Rkf*ar+ z=|e8TTVyI?UL3!u7e5^I+Fjw7pVbepmMn413SK@}7D9 zKJ)UDc^^D&>n9u-xuc;uCKp<}ovKv(EvZWWQwPQ)=zYeC!AYALdkS1x5cZO*o+}KZ%$$wnu${iVYRT2K6!*_^lNY=M`%nz_0v=!O#9vRBK{~ z`o{8%rrdY1KgX^Ap0{LQKpPJh(At9qw6zxEL`vVq???;r?+m~4pMugU7pN9yUSH7< zAN#!iNHYOTFX==oExK^l(XiV5ctM_jwVsY^#j+ficKY{?C&Atw;r4J$NP5m?psp+dPYBhpZ$lyum9(W z(Y;AOSYg7G%kJ2WhuVsH=4Kp$9AP&Z=SON5H{7bw1nPp0u}nLgp)Bl=Xn zi6d(FShsm@wr!j5KR?mO`Pnpswt7cm_=LOR1MrpqDDeF!xf(WYfvEv%({bYs>4$F! z)WJpbc^H?W2(5LTf$)OD_8`^SX66TnmxMP&wtl&``G(*$EZ8GFfse3DLyYP3k6?b$ zpSuc2*pI;P@8AuIpK&p~hu)zK;@=f||4FWe9cE89Gc5Q5rMiDZ?wy$J{fSxrpi*eu z$N|g8YELnK#;x!kd@1}#fiL~%xD>{UwZ)ll$Ovg4VGbo5Z`W`7J7d8*W$;BP@*a{| zzQF2(tO>z#$-2UjbR4W_ALXN(7VK=c|eTGu*A8&~tq* zWB#fK&P^M1icbdF`}4e3)%pbQ9UP4POT};p9P7$+gKtZIDlj;EwgzoIY!m0NF{k;| z?8`i2GbQ&o_44!kUtSc`YyLLQu^TCsvLemA9--b!xP6Ix| zJb4F0%n2nYRG+Ny?Ez5p{eih+FMt`VIE^zPfH_q17>-DQdEV~vsVB-9L3wJOdm@2y zT$6b%I+-Yc9q}xbAr)DJeCWVcI5@2>28|fQD;HS>h@C05>71|`og2>yaRJ1Tm>Y(l z2=Tlm0%G{O)J&ft!+iS$9P2kH{DEt&%HX7`@sR<XJ~+7q&+v(o#!ho_QKFGH?xh#g+PQk;_)7hD9>2JwO|n_W0zyilqDH_ z$^delr(+9zI|_YJ2r^8CX+Z;!;R(hDQM|7uY^>0Q2bI&w}|lbvrf{DcBg`esNZrtYLJybDlPa zNZ$*3HdFD26ghtJkkQI8aS%Il-{}%#ONSVr<8U!S!XBRkjE@h$&Hy2sr>ul{MqA$#KQS(h7cc}FmGjx8 zas`#)jcu52o>Wc@s59H`6ZMHQ?m;%VjH<$0h+!G)tmYT23>XcIezfG&5?xw)rq3u) zUK)+hpT`bSemx82TFOBlXUB)%keMN@EG!Y_nO6;Z?*U}ZVt;9q*P~#Okx-4)1}jpP7q}X_!=%R5h!Ped0t$KJm*aQlhFEQ9)4}PDsv{~M~B}` zc+P?u^G$1*+u`9?af9}u|H%B<@Sbr8?tD23NZY^pH3o1-Bp}WaIO9HJkqZ|M>onps zwamnMem}q&jusPVqC5i!P{s)gl;_0*-cpDykvU(7Jx?OaZ5@_dIYjxK>3@9NIK$T{ zAD(^{7N?mr=$$_5GbJO+qhz9ti~v5^Gb0$--PMTlyoNAqc2ys0N;`xIdq;dW5?5l|S6fXc)^C_FHO2C->Z2jbS|YK z%%Lq|9^nw?lS*>HpdDm#6OY0kV7rf%X$dH!uJjA$8HZ8Rj0GsC!UQ7>hUGHE zUKW|>Jx;_P06DVgY4g!3NqsE1LggoHDPv`EBEG&}uiZq_m>`FlaAJ&YXqm+~fM9+nnP|02cP5ZIl>4(@Wcej%XM}0$4Lhumv&B}2+3|N`2gf{2jn4)QOCh0q+y>{ z<)-Gr8^$v)!GlcmD~UCLj1BS#6qALhxy{5GK?J}ees`u6xS$jjhvvUy$)q-djO$5; zgXc5ep$Y4d-7P>~`s~dg#|}aMaTdsjNn8rYM_*$jL=IR@_vuKKv6*M0%tc`c5E(JE z`h{c?yA^H##IruLt_sUg1IoN!CoDPTB_t(Q=NfamNG{{;0L&B0A}^=;uVFntFmuWS z0Wzl*%xh81ggLkGXTfa9c9$^6waoPaVG6yic1S<(QrO?x(2q-9)&n!TXCN0?!~KV-WMn3qJ@=FtwCekmRweuLd7zzm;sT=fi9^oD&?fEgJ8N;(~BR`IgueB39@32+4(uU9Om zDS+8P?TFlh)uBLY_s-L=%==0)TJKGm%RC4$Nz)lL(oFJ3`Jzg}{A6tn^ZNMfIkj_& zMU_>Abt0G2;qeDHvbTzRh;Gz56Xrf+$SB zvG6|0pur1v;z~!OV=HhDg%|8H)9%8m_A{OaT!SjjXMyuNmrHAN65>40!nu}o0Q2Di z$Xgpx4k%O%*bhEq8(3ozF3$v^V248o$k}ftwF6{>G$}CULvfQB<9Z#r-vs64&a5s5 zmW^;gbLyw;Zj*&b6#vrfY zG#*$P!cOL*5>Gd=daz#E(4|v)FbFY<9Z2pb11rQ_tXlAT9aXd+u=fPWHdw#)4GIw8#GREoe9Y_(p(e5`PpFc0oQ!>w)r=7a}!Ks~|J2q5Qq zE_PaZ3*-WjaS{S#7By#f)__$gu zqnQIBzfHK%py|i!iL+L!mNL*7s98FJOx8+&k&PKM2WGgLfm)@(U|jk-7$1mSfN?ns z<6+|>OnKfNa#dI+`vX-oIq8q7rG+@P)Q}3T zhGY%uEuc&lP2*J%m5elia;DS~TNZibtxb_O>kjcWv>0@5h3X8@lZnmOJbzqaJ!E+=6WNe0}6496uC+5dF zL>4|Pxg5GKn}a_Jla3|I2d7^8(c)Ps_jJ(cOUE*o`O|Z;-lQv;jC3|(103sObz;tK z%1 zU}F^ySMepp>yytW#OIVcp$U!$a;;?H4#WdzKi1iImfa?muy=Jn&0gT;X6#G}ii_eV zk+F@B2VXMEX2;Ae8rt0ixpf8$T&N+pXp}5RMvHks5#!1>32K`YW25dd(>*+9=87wj z4>T?lWczs*$a%Db^3buy#fBCN#Xzahz@o?ml-o=VIy=rlIn6)okrA|T7c^kT4k1Y1 zkQNtS{955f4JE6_1f$O90WEdLWNd~nh4oUvoDF}38l0MgXqK1GlMm~)mnho-%HQq# zStwgSP-~@QjSIl?aFP95J1L5N3@W>!q>sLG2FNh>C&>21GA0i`>MDZgc(v? zgVs3%=EmL#@7IV{Gh$TU=OKHHlDOouzEU4ZT8g-;kfXn+{S2`tJfoEJ$!7|#HTVoxqC zP$tQSP}eNFS##Ujb3@&<^ZKllFQ}~-*nNtr17qmB=8q6LBE(MykWuhkqP#x&N_w@E z>YQjeA9eYT1unQjXa~DR3JcM+NJzOVYHnEDiw!x{%$n(9J@LUs8NNqUzyM@)=Kyjb z$e5PE-Mg@MtTqT%Fqh0VXaE^82Cdd-W`NMvfYvM!&9J+~_(0tPjN4fl=W(FbO2_IJ zH02^!VHG6Gm?@YT4;qnnSvElEQ#0q3qPo?SyU2W1KZE!f{pmil3(YWI1n-e<9-NmISb%n2O_O}EN!7bk5)24y#hB=d(J4&6JdCbVCrfc^8amFGj`Vw7gX3 z18Ga1AfHr&LlTh20m#RvUT01)K@O3cLJ&8jR8dfLL(s*BGUIe^>AF-CWMt21X3X;n zXw7^9-M&H|8zV&xyx%3)vYnBTiW1W?g?{4 znI4jq33FnNtY-?nIf|X7ebYr`KHYreJKIKs!4-9kmNgrq9!E>MRnvQb4ZSF_EQT$O z%lL3yPFyiG1&tWS-Heq^+)|b>ug^VPyG|;@O+JN#{$4&-w(zQ)0dr*qxy>wgqCC(H zfoz%QUTwxyPgycloY1rpSmJvuY3WhhqcIZE0q_jL8J&2J*3sMrY@|{s`!ytH z0F0V2G!~KJVK7#(K#ZlzOe}bt#=llE2zQdmR2m>>Ns3x|=70%6hQ9_=3%ClbKt9m3 zq-6ESSs+_KkZ0v%LCZuEZPORj8yl6Y^9G$Yafa$~<+0Z~TmMbD#SAJFrb#z8Kodbm ztD`BDV!XF8xtL)ZC@0l5BPv6zG7;sdVYC^$A~pb-NYCr;f!KU=eTp=sRwNk4l%V*XQ-xEiveSd?XkVYiwG{%g^3Dzor!9Nt{&M$ z!K)yMWsZTr2*C)ZRj~3J2;&;KM7SIx{Piq^=f(CQe>*mC>6N`ea-z;zAzzDyA8vi0+!g2a1+?YpSOd zH!BQRETW7&omwCxAC*T-ovS;KnJQc!h)NJ2O0id{&zI0(+r?cvT4wI(5nn43yF)TF(ofgvK7$|Jyp&! zyTWh;i&qiK&>Kh{%S~XOIr1~cuK4SMH8h4$MM*?sxQkTg!4)Mv1b8Y?6zNK6rvf}(+<~#W9j@8@0{@4Hh zFW*1Q>kScs9z%!-Gk@5>Pj)9Dl-7Y3whBaz<0B*#g@if}CxiqHhan{7rkKzf6mVE2 zIXfXJ)OiI==r}*Y2^gE63~G2@xsg2#1chKm_qbASzb~I)IYtc8asY9hm!SdkJr zyztBwa|E###Z_dZ6GTe_5i)t)mjN*K}3nSRj zMqvK!yjJyO`vgNVV3=?ZO{} zVgSJkj-ydU#1srfjEpyE+QU`}pES@Ca{jHP3qke%S)sl`-Pk2i6P|l=)h%R7-l$5$nP1=;l`Fo#EN_% zukw!zO=~a>2KE7=D7+7coTO0fsXP zh=<+& z2Jx`RMk1!}M6|Ld;wLzH0mOpT5Fpyf`+(TP9T1V;07SH&0;ORuykJIxh+}(#hz?|`89x=5V%Z-fJr9xEYBSbc~jYi0Yg-~1Vd=D2tPba>$RQ9Xf*8c($r}Kn7lP<> z?g(OrQ3w_+ZV(-64^uM(am?H!jP3_0uZUn1fIjR~X3;ILkjgswgJVZzQz3&xUzq>` zPf)*lrp!uZSb`2}0EoFpEV@hp!E=K-1-ETtG+f}q#SPQguuA+&o#XcdTvmyD34aOZ zg!XDe(bYA6pjrif7OY_UK*3n3Fn#Y> zara6zVvhC>{$UewMLzUmqW=M!Dj?VxPocut53e1B-qsL;U(`_)0X1CSqo^Q0pdm#I zFpUM6sPWkMk}@*ZfLJpDh~^beBf$?OGdXh?OwKWN(L{{^W5r|met>|A@S!ak@YU|6 z?%~U8_|OL9cuk-gOC=9aqGTdia@e1N>s|zQ(cTYc2XY|cZ}vO^G0_76&GQzB&{xWw zaqnK=NRERPBIw9tdeM*#CB_oqOSD^DD%DfH;cyfxa5}<>J~Nzjb$AKb-tM z&MaIfAVb5k?|A~2DEwR&euyfNE5w1=@#}w-02}#0oR|F(TbPeY_(kwJBZnjW=3D~N zet;7S`1vq)aO#)j9>3)sBBogkJ+(XBm-0I>bo#(VKu64UAbKJVm0K)FG!gdW5~IYd zLwHi(<1ueh9On0U9AvtbEsYg8=OK#W2k4%Np!&DF@oU$@zd(wE4s{z_wov|1_0sw2cL+yQ((E>|yP_F@iKD|lpKCt~y1%9x8ARU`k zg7f#x+rMq$ppeD_uMn5?)^0xF8PgANCJ{gQTfooxw$CJ-Fh~qH8bnAH(`x;%M4x&F zfEENG7*6^MWHuU^!J_67q`HtdRN2_EfdZU8V*z?CKyb^T!~$MI064!w(hm?K0l#Dc zg+^w3_h;a`)Sj3kb`l^#G)(+35DUkbVTd2uW)5LKkiCF44kWSBIO0_DKq3+A(QlcU zIGn_!O6gdC^Dfeu>JQ+4va2ScbuqEtd*rU?#;f>wBPHLS z%82;~h>8ec8#_41vOfXW<#qINLPynsf&uNbL=ZA84mCn?Ji(+zb^)ZVKqeP9{IHUvtOA)(`h)oI9KIhQ76N<~1xMi5gZzC8 zVO>*?>>%j)L9PPpiuj?S_yG}}@bgaiMebf^xElEjIa|=Q6nPe=x4sTG%b_9wZ!nQT zneffWbYuAe)*#}C5f$JUa@+gQ>vDReMx$pVLD3%_G&J;}A^QiY9gOuCm_kU#FE)(6 zWD34Lu-6UvS>50(f-krP_!JJK)IKf5#rgw$K!6W-CeX8TkUm(~(ZPA*(bwxMQVFb; znc|D8C6m>-1@ydjuR@a^{vz_xPLABkt!&AckOUs&upmPn26%Hx(pZ0h=7-=xlnvC2%X%}yKF)wXw=*F|(xFK}^yaD$Uo=&NqLa$68~gbs2LgpPTX zZZ2S#0+XufgY)37+@SMXcbI^I*mN1O%p&X3et?pP(A9)HmlQ<%GjClz57c|%d3KjM zF9qztCW&$*h^N4tO{WC?zJxEavm@h{Kw~jq>UD#-wB=kNV!}>_2%-7f1YezVpvCqB zR6BsrN6UiG?GpxRT{%ojS|O{Q!>+(WB;iMG4%cJ$iACo?-Br+BR9GZbyMtEU2iBi`hMZ zmzeQOh2d9g0MFDbjYBU}&tYJfm`DJva>p)(f!%zzI=LTU#{qVh#>Opv-*lDMHFMC_ zpi>##kUeBUbOpi-UZT+ce!z@XW7A7iM}8Vz(o#{VxO#OF`(R zX`&$cAK<+Kdch;l3vxEwFT8cN+=i!IJ;z0`FF{lvDvTSfdD=+z*v zjLwHQPY*OrK##Tc_&5>}bxsqT{14FC5Io2>1H69l?+?9ovD}m`;c+A(WWdL*DSlVL zm)M-E$cu7UBIMS@oafyP$`)L}wuNaPwQ*1apiCEq-JDfE`5$1a0d^5nS-`IQr2Ca; zZjD`l+rcpea2gXjwDZ7yGeH;GorZEQpv#Q8GKgA@A`?iZLNGkBm2V)Ig51-qlX{@#5_e6 zaL?+SiDD)tv`4Gn3LP9I+9K9bPmjQYSCo<4GnBz+83b{B~jnl`OZ2$)SW>a%$ zBzk@nV>P4XFTHj(Deom7QO#MNt zdEi&`9KRnRg8_ayqGPt>cQn0~*5&bj7vP)Ori&1i@Fg|IBL)F{P`l?SC><7XRMH2j zl)?2ZJ?6|2(6;Sehm9W9aP;u*{KS)FGp`Zej;1;xb^Dv6u?jubf6S3pX_hrnimj zqLvFrcfSwoW0mQ^AXn?qS244CB!WDv7gk5g?zUGu{ja_OSp7ITq zHO^8d5=8^Ms!eMG&qSq=;i7<3qF{nuBS?vJ?hw`+;5qHuf)mlkSVCF}9H;aHTq*#s zfscpi9Smlrburx04S~;lClWS*msQ|N!p>D;ru<)*K-{zc`xFD%Dds ztBE|maSWK?`T3ETet;r{;8iq+LBaZ9CM&H=;jVki{jveG9BPtPOV2kI$UGyB0tP|i zm$h*lA&K|k6C-+sE6SMVgkxZzw5#``6#*|1DEeMEHJC229W_DrtYCugmP{e?jID ztWy}C1C21*yNq)@XdWqdsNLB;TKwnuMVrai!g_oqADO937^Im%h!_j8XL9B*_?Z)K8z=YFdH+TeB5JX9{TJ(Uy2H8Iaax{hh6T#8h!*srdI{ zsQ|W8%0Gga?I^vn)@Q&~;L-o1bFbjsL&e6ZmM(H{Hj*{+q8DcOsEe2ANAW#s=Nh$7Ky)Ibk~A3JzG7G|n{CuXX^0Yaz|;mn>(Cao17j?Glw z&6Ia$vYDC6m(5f^m}z08Y3IV<+NtNj#ol~3QaWsAO6ztA#zXAJM!TG24KJV?>RGhM zWW3$Mg@SKU!X2u7@g9yY(@T($cw`eqB`rB77@;-FQC+A)*!!#cxr&4c&%9jj^n=Mx zl2(TLyKyDR#$3ChP}<-3t)w-30e4gR)^7UWdFU1HrutzwwX@yS{%(8-B!;0k3c~m! zpS+#<5aY;0VB)B^_I5Wx8-;c<3Sdz-?@r=EXo<@o&$B=6Pd)oT<#K9Y^C5mcW&iu% zw57m`NpBD0KPBx0&viq|_q@F{@%G}Cj(PiI-vr~D5e>^c6CT?+qn z`}gCY_nyVrW3f~=gTwU2BhQ1Ge;rIW|NUdog+A!kmh9W%5Sge%#&)@wIAAhxXv)vSxZw>eV``17IyZ;;D|Mjo`?|=RE^5@ro`Ah#x z^Z)%Y!!rZcJ^=gt$DjYx1AgrP@DF@%JpWJnE#3~T?b~j^pRk!0bh&hBD~d^JcSGeh zxi-j)FuIc|5&Vwqr>Dck9=>*WwAb|0M59MG33<2JLp@mRF&-}Vn5T;!yj#c-1#s^) z4%WfZGcpP$)#<%)iViIj7&pTsm5#_m3Z^bxE1a0;Kn%*blCbR82FVZ;ZXXv!S%_BK zto{@fyrn-}yvAP&-y@ZqBkQ9bQ~ciQkM(=2KkboDM@Z*mr1KQ%giTH%ar76nRfjDk z7-nwhi-!l4K+CJ?Tktn7qj91>o7q5ug81O8uP)ZKnIZp zAu!p1J<`-7PssxuAb)JGRd8(j;|zEdS4I?nTFiE*?|INSP2g+ags*-RzP5+29l%%I zA$-Lx;LDOm0}X!7Km)B9>PL=h)I%-u5x{h$BV}hWV)4gY5g4kf4`k`U`M}cyi?X}; zRV5@e_I-T4u}6@uQU7v%4pB6kb=tqT6J=^Zx~6VJepBM--xDgJww zDeilgDSl^};*Tt2$Ck0VWuyt;EtAPmKq?L>HL|lFyqpPsVCkD0#ZT~s6ZE%L4ar?F zcO-So7{?&k(`9jd{`T5Tc~?i;j(|hSkNTC2`26;D?tV0;>n-uc%4Ui2%KFh}ddu-u_#^)9 zfv=qy;G0)u`Q{Z__E%&%x+2T*6!2(8D^!r9lFm3kqYd3FKyxvV1uDR zB8(sucAgDV8B=}euhPT&>y2$KT+vmGyP7BUeDlZK(!mFm>o@Pl`pvts?(fEWbT`)H zyRk0rM$G2*6U7-@Cg^oUv{WE9|19?bouTLeWut`%l;G6qMH4{2d6m^~US;*;RW=Q? zYjn3^cD=psj>Bvs;x)~o`ZhEA; z$YE2&(HO(RDXQE1^MBt`pn7Bql-W@H>-g{U`#O%7Yi~5?zOXl%i?@5DIXmAQErWDzG%{qjJItG_U)~!W zB^7|_=HX~}H1Ga~cSrk}_uu|0?srEsceFd2**r~xW`)*w2L-v`$^p-tdf0nO;GY@X zadkolcAWZNYnz=0iR&43Q+VNc-B{Wmzjd;(+s%~E`ND1%zqhc1ePv<4k6LWs7Qf)7 z6m06gu0bteyEDH<_vLl{hF$pZ>EW7=f6Fzm7I!tT77uD(EgjapTAJ6qpa=nD1W;jk z@@#r^U?BVP{I(p?6i*cdGQQ#^4|>gHb>{Dts#^M9sjB6#l&V_(RH|zE3#BRoQu9FM z;Mgtpjbrx%njaqym+0S4r{s}I_L+-zl(}ffnTxi{Tx5zwy_Cyk9TXzy4cWq5Zf29OMP(_d#C&$D<&x|8t$!QZXL`f#*bGFLt1^jqUvi zMQWXrWGH{?1?0m%T%Ez$w_>{f`ftT_{q^-Lt25ljbp0RP7i#?gBjka`!SW1eG2P(8 z!36#XDBnLlT$|ymjYri3ptvhOX??7p%@bNsYKvwh(J1tzs_ptRi*&1)HA(%5^l z6@SDt-##5K)ABbQTlL@GcFVNHgJoLX;WDk$*eW?_*x#o}eQk7=1on|yDDQWCPV5o^ zwfKWx@t~JXR%mnITcI_6Z-qAd$_j1y(+X|pF%scG!;}Eokv|2$?ml#E=zR2rNY)L> zj}J$y)Bi1Z2KL|nbClHoX&)s8f7lt=|2fWAXXHq63Yq*lg{5|6P<(w~o`#l*Ch;k~ z;z2JtUY`DY{e1oR`&N+D-~S6iQh(KLkktPne_?q><)#G4PMtjXvwpt9J|!HWAFymU z4~Gl%%iVnTZ?BuL|Mq#eKo7bYfI&AkxJ`3zAw;}7{)8Pch6RaZGkp64L{@| zc&JxA@FkN4`n_(0{`=MMEztCp5XoovX@RCMgh<3A+5lu1#rJoOUtOR#SblssT%bAs zEuWxy7b5v`5F+_<7$W&H50M}>ibkdeHmS-U(kjA~_b7gA97ColOYa6RdeEyTi?n_( zQu6hCk&>@piIjZ%6e;=kg-D5*U~WgC?AB<@XVu#JUakEA=grgMDs6wm^{-%V7c7}O z2$swp221AW!4fEN%-<*K4cDzpDHJq-Qs+wCQjyIT!0-m@Tl$OZ~Jexra8h2%wq3klQf8=l{++|S85qhqSu%0c9?qo__;EW ziSVPh8vhJqpf%ehTM9Z%yyOj0y~+FMf80@lyPsXPzPDFlC*e4Yv|LRX_rwXN{_Y+m zP%9>oN8MK{-seCP`u?M!rwk{{N&dZ?@V<~9?E#lsL8^~)B&2L7Vb~qNF4~-%&;M~c z?bh*~(lhlS`nUVvNes6A-(Oz;>tFuet?~c)*I)ki&;RwmgU(2et`?){OFukPt{*>r z`0xq{)1p&5R&-fzdl!EMen9^t;0pZZKmVs+|DwO;zy1&Zur22Q?fp}}9xv+OupL6% zU;H2c|M2I3`CtBX_c2}{^b7o_*XKX~?qB%J zpa0{(yua+>|J8rRfA|mo-~Z?TgMWHA3n2Rs|L))YyQOiWE6IUVm^EW_(aOh9A3nUI zAuZ}c_tVH<>OUKYnab}N=dC)(@getpwAC07U82ux%~7kz8_Aqvb6_n*=Um8?Y#X0EeE6zI4HL{k;>nSgQ~A5vG_T(@Osh3_7Y0X~PtG;C*&jZA`0$GE?{pU$)BZJe zAIWTIo$q=BrLK>(P>cJ!x^Vu-?^>s|ZV6p24mrm`r10_cx7QdY_u72|8Da~&8>fEP zAE+n(Kr6nK|0~}(trky>SO<1-*W=6G=i5tq;EORmY#tDzY-zO**7>eKQ1jY4)+6%v zf8{%;j;q$lG1lPby7w%KvNY&L{G}^xF9JIA_;QB#Y(%+MP{+9#$d`Pf{uMBnc zSA9ybhOZ2Dq((hUkcnbA`%^A3tYxWGtR0*b;m)k@U`#U>BS{`2VdSYx2DjK9+)hyXWr>pl(WXYU?*T;MI-=cp1~BFoEt_exQ1Vw$cLs^_xL zvGlmxB2#-e4}aVl>8cQ@bGC=+LI@fsah-^b05WzXC5bi{G0s%uOho@ug190EyO`sI zc#mTnHU26{Cuh6ER5Q3KN~NqNq{;BrhF@O-7YX#`URs#}KP;1EKZpL;f$t)S*alL| zf4U3Od3^BT?@nYw=)Z6U0RpMxyWQ@Y6Qcn^g$HyR1@eKyjSfzSXfa>Hib)1t1j42b;ieL(s$cXX)Kb&ta4mMsQ# zLN`RUbeBL};i&-Y3S^cnq1tnw1Nnxu2#}4t8$9SJx~w6OKLI%g+`$!5&^hhn9`khp zcXVa=Cbe5kc5@eQ)G1)(KrsYjc6l2zI{x$PU;odyhEWI5+F>5O?0eM0TCIy~Ld6|} z;2iCqv4R|u{@}vUAq19^ik`|4!R%Q%hRoswi%s2!yn`xNV!vwc-+*ce&cR1?ypcCa zBJ_9G9s@K9JbH=yF_?JtfxQ50y|L2(N#@{|sSD%z)Mi+mS5=a+IJw1k^*e zg515WsdL1+NVcqdoc5)p+XtulR0v+c%_S9?euKt~M>aG+Y`8XIUYgLZGNC!Jo4|z8 z?OlkqPT4SFs<5E<>5f;Q!GaiR^wd9zlq&O5(?gp9_8@!-FG46MQ6$WRb(1BIJM~K) zj6{+EL18geo$K%l`kVAk8X@s$_oC=Z&=L^~TD;5B|03~FD|b*^lEs?8eHf#b&}O0m zv*^4a{sIFglK~?VYScnZOA9Wf(td@f<}b5g0mUvXSa$W`5bP@k?2exm{(-Qd(SpSa zEr@&E)0zQ3U$k5=2T8 zw_ta6u>}KyN=x*fDvQDJrC>Em**$c(vswQ1Boi7IN+z8!344YKOOpvHyh9U)wF%?W zgzhR624DsgHgA4!!~6vs#*T@JTp4o%eV+u>wfYqUZe-Yy_M|jc!#QOUxBSvq!3|l_~cSej~abKd3Bo@?0 z3=RSMV}jrjZ=`NSUui^MhW+TK5j}O2pg28r;kYaNf!eeYaTfOjGRU=X5j>l&$%w8m z1SoSdaLE)9>PAG+1|w1(r;%IM$a&RYf!*D$2C22%C)qF-vJruAm|9s;HViT$9d+KT zb!o%6a%g_l*UM}O$y3^}+TH_?UvX#xuEPwWXxPwVh7F@bGfb;=XlSsM`l5m$q(cj2pm0?zZmkJk#w;}F0FsoBO&qIP4vkH}5 zx7h7{3vy@Vy8!3axchJpf*DgBQ-xBx!DKEAw+}L1##YUs2j}ScX1Y3dpZDV$JUKy= zTbl0qD$`|<&C_%#>|L4k6<5YO+#n$J9H#SDVLIJ{6`qW1w4E^o><2b&r_F~5M4T~! zt+6I%U4(Z5yKij18qqD5MKKJcLR67i9wez?zrs)wtqL$7d*dV*n8`f-5D+hYev$!8 zq8~$PWSsd}2x8Kw0ed(ppQx>m&lHyi{B@ZDYrkY@z*^hhfM4-rJR*2NLj$@@1G>I~ zaWLZebUZPG@d|X5=O~fUL}>VmC$%nw2XSIH1?UJ8OddtYv+sjD2sCs1rM@Bzw6aN; zWj{azB7dU|Yo;4x@tUYdzB5rr418$+vi~NSDY9h$c`L_1^X3cjrmkq5bV;V^+k?`7A%bDW1t1$qWa8! z^nQBdz+?`Z{sNN6fmp(~EZv$gU#v8rm#YIick3mTKTJv8N(E!MxiVLi5@JjB)02$X zBK?=>$R=Ed@lqSc^Klq&9nEK38n0euJnF~NdVAUI{#$*;e{pnQ%xm%k(v*#?p0FN_ z$LJ4oGDNNlO+nle0)-Ww+nP%VkW&b`haP{nq|Q{M$=nuYYVR2mr~1R7OYu! zpl>PcE%+7p1&Zh0gkC0uH_=U4kmN+bt)mgHIqyIQ{f|Z4&jRJEX~7FM$&T(08^tm`CpaZCH9tL)d9!bC=|rGUF&Yc z9JQfaS)ZY$E^QbtvmwaAVMFlj>}*)w7i?G&#dmK|dR7=!Y2b*U3F*V|q=cSTP`O+Z z^xc9faJa=_!Ae8xaDj8)W_s5qXN7cB(jE%U19Q zaSJS1lQy9zUq&rD`})m*3pB`R(SLrD36rBE(_Lc@vT-{*_*cV*Zd`#nW9kbwEC)I5TEAe!(%-@4RG>=HuOR~qFJr+-XH+KSbaLhHDo(r56{@VTKV%I3 z!GM`XUzQBb@STjpVL#r1Oh9|8;J|uC#>4Sib<=T!W5-buG9VD}pPt5)Ax+n~`=C)y z(?yf%VjOp1S?6JzTbM4E-`uRa?oi`v;l78}nCC-5_ufp(f#~xr_pC-xQUgvYsuBIDM#ByiO!_f; zM^+};KQjqhlP)du@LsvwkdG_358I3=eVCfC)SOq=z;coKLK#6|UZ0*VF3tCPmHB#@ zUS+=+_DSsa6&EIl1p7tVFBaMlX(%YDLLRrVjBAk>V8sUQcMV+_Pk#oP>4!x68b8+i zSO6VnRb44Hrh88(Mfv8`gylLwUK%hS899h7HTU{VKYzxG<@^iYUYl8+HUxt!umi17_7J zbO2Qz*s!!j8z$N?P@*$YrzbM4#-IYV2fR$2dV$FRDFt;ChD;Oo(^4kv8$6%{Z+t1z z*D#?h=&|i0V!J@!_tTRsSfv6*6X}RW?q~C1wq84w?H3k|&XyK@y3B&zv8M%--zT%9 z`-%_Kk55<-W7ViiWEij?i!>mHm_=k@3?oX$=ecMPDyVntkIanfz1mVzW%>q{fo_x# z(B8s*Rr|%3AOh8x^zM#NzpVml`Z6_2$=!ZIH&?$HkU=e#Cc}{S?UM}HaGe%*zK8-4 z_KQjO^TQc>OTEqPH}_v^hxgZ2_DfkYMCvZx-h5wiUt(qq;fMKPJ%-60;tK20Vm8dj zNFgf^qasB~lu0W|bqQ_1O7UE#$8^vA2@){<@Iw^XZ#`6fFhk!@y;zig%a2rz>Bckd zJ`gGtsJh-mr>;AtpPumR!^E^5d#obX1dLZk7RI|UjJG^KVLob4u_@Z*vOHGpzN)%>*b>RSHtBx`f}P-Y zVEZ{Jgk`xNbbmaypNral(K8d{Mb^+*J}jgTam>`+KFNX}>W_#laAKs*gl)tRZWN|k z3TqR_g$aGBmzl5)FSMULes9CCcrgLvb&+jCs7q9CK}Z?FhA?4-q0BJ^%&aTr%x((D zTy{;Ps|XiR9mNL&IY{?2Yy@^_>Ew(*G?igN+?p^Tin+#2?~@4rO<#qr8(j^^{syycNdPV`m5QqKgZLq8eQBi#VVx1n;7 z)%+V_KW;gLbQb*=@qkw`kzg02fY?4g$$9~4wZd4^F+oe~#Wt+x#`PDMdF;4o?JdET zbd~jLh{}WJwkN!c_7(R92>=w~(0HaB%;aUjdh}nIXz@)3G~R#^lEl9-kCqgz z{gQZn7F8EA`*B&6z^_?gJoXTF<0V;7hbPo<;W?SC$6O320T7h3$>QUatj8`!W)Rum zm}I?@Wj!S2Wxe&`x#;S>)OwA2Z^#MTK8qdgE8a`@*MPB+3duP-pC}bzdW`Oi+{(%b zFf$VazP)ifnTn`%9)e;oCPzkdOC)lM?+cM!)Os^R5g4!U5rpHY$QzO>=o~E!9nkxY zlCDP`7Tz!m6d#{tJrrSJy`=kE+0W#>tXJ{~{+5}uxM=8w@h&St^n+coU&($SIfSox zFBtqs{msad)FQ3t=!u}gjT$Fl^kwLm$VitF2eN(z=P#-VhkP>H&6w{*9O86^Au8$H zG*X5elVl~rfarjv0X=tLWd0gGz_3|ia?Mjzp;A5)HVf=A{P-jb=0N8q(y?+@#^F8o zUv4A#3+v!v(a>A$ce$tn(cLy=5^2J=PhiLJ73T$Y_=r3Jv)MhwMwyka9Ig&rAdx0y z@-wkGl!NvkdR<&&`XeJp#TGVIjoRB?l!jtBlc5DQCs>f8u^6-n+=GGnYeO;x7RLW1SIv;Q+aaOZyi4@dU;_#yUKnZWI%yc(Om{K#;>?8 zIDk=ovqt+FXC$h25Vkj}s%&#dx`(ewW%Eo3-uP!3wkR52uX~4|rKjU;W4t+xM zy(Ud{c}z&C-##+89c;8?Q&EBjr2SA!u4q`I&9Dq^pJc&m^k0&i0yYbl;lHGj4#yq7 zvf!eVx7cg(mst=CkQU74z=Ws%OJnakif^)D;RYt;tefIG0CnXqZq=otoIk)2t@#GAYI+*Q`I?mEzV-G|@D4(Cn0jZH&{&xZ9l z7ML~YP?5mKpdZ3G#(rqLM*C@&UX79dllPAkizLcNW*h@{1M2RsRB(c#OP?}UuwE`Y zK7GF&gPPy)eaLQCP$Ws|3jR#uj#iER_Q_G5+Ie~~S^3tW;2mF<{cPNSwvHYaoxI(C z`S(@!>(Q?6zkWgQ6ENqUycoY?@PIlyJH}wY#7ttXl;dWS#q;#+*F4Qf>jgATaP9y- zRBXWvUiH4f7Vf}E5ChKPE-FcaxfzPyr!<60LZ+B<)9wol5-4g7`%xf{8#=Snc=I6h zd94cmk}25edn>YD*f!rfcvy7tW_!smml<#PF0F)p>~P-1+f;iGOmc8mHgF~vX<;-m zP>ce$3(R~X1ScCVt%pPKF`!|haM+V0eX196Z)m$=Mh4y|!#4F#!=H(2Z*R0+DZ2Ss zIw~g51@1^yT^C1fxP2I=YxG6rKoJVsMv?10z|yNoASRQofto2Cgc@W_r^s$yEV zR_?vrc=Lsz*&y7W!H*7XoVhd^>ldCS;L2V#d`J59AOq4wNawIEXBe=!It*xI_xbhF zxoF`{-GWb7SujClofeE?A3EeOI4)=(L`8uHjD@qEzB2eO$Xr)WSED2f1F|G5rM7}* z+(%t}sj7c=REiwyhMs9`#not!P^Vwz)n{Y4L^Aa1z-GfCttZDt=V-+`Rg~j*4VFMN-5{GjO z@4igmh4-EJ4E@x!3$hNz)@-}h0p@8zGR==#pCd0kESS6F5P0e$Ps8I=H61J*g9UL0 z!(mN4Kb^8Kp?_N&ZcU_m&Fzy6h#7FC!-4m2Qz#RYyH6 zSjQ2H*p?>T=9XqAoI5ble&u?V3H!>|CFs*Z?z()zgW*K{a4b3c$FW>y6WN~+wtwWM z8iPM*;CMm?6LMra)q!7@h^y*-POW^bBvnQe+`mk8Cm6669ili>gO@8pz9AT}%7?KK z`fo^PBAgOAetMDtM^VujiH;0x^-RB@<~Q!Z^$cfRH1NWJ+8yEOt9MT434i<+{Q6N3hK zP8{hFP5nDTK2sI;_d^91EOKHH+qb?EWit!y(~~R+LVB36G8_%cfFiu13F~MFUdQi? z2Ht4>NV>{^6$u6y&}^^g73&wA7}nnb2W66wsYZCrS{j;a(XhlFC}|pFnix4Lrnhj2 zX6!()6>tZJ;bmB%S88md#{iE&aKE8Qo|vdQFIXueOHdDtD!X|(^#lthj?L#HM+Vx} z7+#W-gTvtVNfyLX!z$wVXo!m8F;OTBYO1NW^&z_G--QX|RVM5avF<@jc^^sCFZeMm zzXy?rK#7Bcxj}mn24u<%`qq$WVqa2%kiBQ5X9~Ab?iHuU80kyO$bbNmKyJU_;(-(* zz?PNX)PH3jqKR6LaBUL|$SOXP-+?SZS=S&s48bQ>M`pl2J;{P-6^8{G*u>GqQO6c6 zX|v!ul329yf?)BoB1Gdxfd$*%k!fFWWP~8 zMa6(yk341X6>`sJT58yvz1mQ8AgtrUWdC7@p(KCsbS)S(v6!JtC#@B8i zWI5XoZ|v=;6byDHEtk?*5OsO=rA6zmfo=FQ%OOiCUWd3h+!ve|-&ZoKrnDTgT&Rtc z0;UFDeQX1S+W|HXj>D%wIr7Bxb>q2No2M!gTL2TsFPV{1r>2XWQ;|?;zhn3&S$K3A z1~GxFz zH01FEUG#klJGn3TF0OFz^*8|V7^fk~y1^D`=Ds*<&iaG81(Pv=NW5HFLRY^bT??zD zI+4|yPi-s5c^J^Q`HgoX@fyUb9O))8A>-A*%ML&klX)IEnas#(`|(Kz1dH2)x*bg% zGF%CM7|^F-z-1C9E&6xOa(cbWfY7m%0rNhMo%|Pk7Y@OTGoZ2IhqZAvbD{iRL{MnF z_#DgcXtzg-xgjjaS1=rnEVhYC3roO@(*v%4Ae${Gbh!x)xwkR$$Jm;i(2v$NL0Nu=s3Pgfe0Y!rljjyprVvG82$mWx7zPy##@?0|oVzZ*Tkz#F3$n$}NblD6 z7Cd!b=-PAWE5NhEXy#U>A>Clu2nwm%*66r+Mh`d)ajJ5--74)zYHJ+A(w1v7U)4yp z@d8Acg0%UYvr_JO$|~+Ip5dR940p*zEjmV3N2nX!*qQ;%vYsiWz@fB{pUP+) zo2fkdks6W*%O$$a=-#vF8=_E;+)iftf<6*VikoAn4`YN*CBy!|v7rB2( zE>dw;>Y^hPwoXa(n58N&1r?@uX;7bTA7rO0M;H16k0W&Vab#EA;3%l;3C*28{-@Y%~yw?)mGEr6J2i8@{|H$AE{^BZmF?SzB7*KppHR6(S~W>`wQO_K{4qvalmhJ@Xeg`Gc!)<( zNES)k45E~K^-|uAH)c*4Z1w@ZZMr98f^CYgxaq;-9Qaa#w0#A6j7kLnP1S*F3`18U zc5q1WVFp1)u|^jnP**q<&ZgJEhT7;konA)>>)C)#dVvTwq`q+){)-=y$zj(24m2|i z*Zqz>x=9lcNQAM&u-?Z+ClqEtQ%gaG77_{Ur2%Uz@JXx;Ld-fAuAzFLL==h(FF}L@ zUD=ZK5Q#G$hnPY);*iqS-_-4PPyOGai!elh%cB@aE46*=z1Hf-Po>t2RZ2~c#$N^N ztDr!fj(6z~NGez+Yo@=CV5W-uN>W-kV&m9zo8SjlOB7u-w$Sa9PF>SK)H?}%^p`Yh zmz1F^%vcUU_unSeO~(XyqgM(R1!DrgPP@?({UGIidNQJM^<+e_F0Ps!>3-G*W{=K4kH7!bSP?C`$x!u7b7>wlGnUqdf2j zIR%xYF7_rpjf!gp@YOau>9H_ZjlLHRZ=$CUlliC;4TCir6m)aw;iWDXGMMD8FaF(X zZx4gs#m8;T3}!>U*0T}x&~EcK>Tb7`;#GFTm>ta){62@d?xKEB0>enRdPe{Tm$9t) z@lj@DNSiel)-93*NwZ3`p%0J*q2YtsATs7Twji@X9bD9Rqsr%0#I1Dl1IHX_HVB8q z5zR0gx>ZS5CQCU?q-|zuEW0J|}w2pB`no?8G155QqnLKX`H%({$we#BG#5m<}V_*tNrO84u`V zfgwBsI@kvF9&9(dNwiyr-Pj9L)C+@=*v1Q~>Bpqohe1mhRZ@zX=d&l%R)vPdq21<< z)4x-7@%1XZ^~D-xH(&Pm)w*#SNVG9)=V&O`bT^=T_0yy5#wi`IunQiE*J?)77wqOM zxph!Of47_O;X6zPFN~IJcmx*#PCjaDL(PTEf&nF=I zZHDj|cVhCxX6>F)lTy$HnskAC5rbLXyyTCk%}~-!`eFv>fa5J$*bn$-ueXQ6{oh7& znc@KDG?5`V-fHvq=-1cMJ-e&y6_FN(y?kx^3+|*nddvws-B*RP8>A|_eHg6|PU(=K z6H-@QH=&QT7h1V0H_f}nI0IeSutkfZpk5$|fwZPPc+O;TVA6>w99_7e1Hhs{z~20Q zA;FdIN=jq03v~5lOg;3suxXxa-VgoyTtc*~43^My9wo`O4*^d4p&!E#O-M9RPU4m- z`^V97WZGXrCnGs+U^CMm>v7dR`l9WF%@W$EnT!}VN8xuZ(JNf&fKgG4xm zmP_sd8@w9r^W*5(jM{A+r9@{~ep4Iz<4(#(8Zv*kT*%x27^kD7jD&}=U5@3(cIvwy4kBLM}!u!#T# zLW7?kx6xRk09uSN2%+ItCRz>ojKrb*&1&dnr7SYK#Ar1nFGYTf#y(oD$)Lz6%2Q0x zO~DZYD`oaIJ5r-kf=>(*v3S~XuaAX>=(#=%AOliWTIDghe;z(%7DH?qMc>HGFRk<{zmU{IUJnV845 zhI_4X380w|ks6Og#?fxejr5Zl_6pQ#j?TXs5*_Go5%$gFu$XDu&*_)bcrwNXkKO)x zC-cioPN7_7F%N2tNM6Iez4F>#F(@kqr6w#!Kc?K`?s4N@iNj!xQChP)=!YXshAk9H zzWr{r%(~-n@t`r{M>UUX0{2d{Tc9?oNqMVrtre=MK!t-un>ZU4T_Uu8AJpGT{OJ(F}^w=fS5XyjNc4K>C~W6GbNHqNS{ z*37wIj9$q1_e}ruZn_6GSomPIgzUdsb!fNBT4`jPD%EH-$bZK+IktnLUk?H|uob#0 zF?m-Ri8G^>Gzz^L?d@S0jU<{$(M(X4UKz9_tL1t7@=HI|L%hmrAq__)msIywi)*Wu zQPDt4WR@HjZ-P%x8^>SOR%^;SwBh!9y8VteZZstKar$lnTjlwPUTK!^kw0!a z?4q(&Z5RVob5uv9)vPgpqEiz6tZ!C(dl*(j1p&;>eS#Q=cCk8r&S)))KO4>cX@q%lMI&eEhjW5D@^!**X zPXc4OcjHkdp>4%BNP>}4ojo_mB@gLrv@>8Op%3Bx1dJTE#E`2`_gk4XvBvPW>E0g3 zrfVh{MmhhXsI;4oo&P-VQhu@YWUp5lE=Sh4F;*Dc-f($sI1(c=g+NQ?qNe%+=l}L` zlU0c0Y(>Lx9OAx>!;dhWSJ|c14~7E|6lnLipbewwjzoUw(8H0)n2+8AH!*w5Wjbo~ zVO3$cHQ|<1V?rd8%P`%C2mQ?MT}O&YD+UNdLbLw{$EKV5CeOcnEdF$r=}H*BNj#9- z&g%wcV~#;ROf(%i^pLaAJ@@f()1t|vcToqXcZ%H1anE(Ly?c}3SxNU)`PD&IVY0&H zFxq>VqGtvJ4QsmUWGb|HwZ^VkTBf`ksoaCS%Qd~apz2ElgheZ1p1J^rC0N%{8W7FlR<+ID!&^p=?S z^Wjg(Fg+JZ!4>^Gq3wWTc0WW`TR6~C1b*%j)p?=Z-JJXOls;BrLZfTRTT zO>r{W_(K~xPUGfxi5eW2+5x`VzSpXlj!j&5iiG?~U!L zXb**|5SjYe0iEtEwkCS;zg|Z(;E3uWAVp;#lL(R-Q7 zYka;6u@)ft!)}9p4_}TaaeO3T)XfG1dmaK8wL82Rh$OAMfO)syYc1}Y zDlGu>J+u*GeqS}p=`5q|aBZTwRzW1HYSg8Kj#Kg)!%;^qxZE(uokk-?O`jhJRR*fY zoQ}+VAG||8j@ag1cCR5nh3Bg@hpJ0|GAr$E3Gm z+9x!r#NC6PiNqak%udkPZKVv6zun{DLqe?-JX?3D_Tp`NxOxoOJdgS-hmbA<40BqB zY)B7n_kf+8|6m)UUGU3{wm0jyd+|NFVvgMSuquCr%YnD`^<5B4E6#pPo4@+P@(>nIXolu zpk!7)KMvkPLzyR3=rVyh1++lNM}PT&oM9rRL^R@6^L{2Kmy@I4GJ4`0X*oc~gnIgQ-!Xs^iVGb}tmsR@#Kn2H}pi7n>* z9b8Y)MvU)4LSE%DQ72#?79|y&EbA%uJa31(F0BxAR<^6(H=u3X$9r{={(D!FHrDPl z8R#N;brW44mH!e+ToMxx8mYuguzYRwk|IA0McX^llp_U$0i>IUf8>w(Z^yvpzy4l{ z1@mfsIYtNGtbG7;#=D4QRP|VQqx%Y%*6q{qeNf5C%-WGrA0(&ve$gUqt6y2Oj}K$i zmD+<4X}DubEi!e8+JVC!3yJ$Ga8fdi{My^YKT;fpgOXW!#6$es59GrlC|@u-7aGbs z`(cp0ZY~Psiqecb2D{N49L?0?VUlnHoe#RKR3u8H=K=md>VPp5a*P0rtt7E9-lT91 zeHZ`vkvpYwF14|6$A%5^t@wQG&bA*&odH`<-pEF^=l$KeZUhQsnM1x3qB-iguWS_o zKQ9=u3m98Gm2A-#1$;vq6G!B{$Bb1>OtO&!KnUXmB@?Bwz28Ou`EelJ&s{jqg~hun zMj|)yf7>d}mQ%V4Vf32|;h1*_mvzeyRi1$`bK8j6k+9HMIj-D77OTLqz=xpOw9Lzo zBd~!|ZH*8^r-uy z;}^j0@SG^~mX!{0_g$|77!7g6*XNygR{Y5&hbjC1S9;{pS;0tRGBn9-1)_^cIHFts zPJRDMB~VYbIKdl%C=Uw(j7fb`#tG5=;{dpN%xDG*E>+;bs$d# z!hP8sAbfJwamwfsP6Rspk%l%P48aZvVHL79q05SDsW>aEGJ*+{NNE2CzR+m_zHHlb z?8=kMrj+Sg{rtG$8?b=0ac5xi6SU3MALF~a=zRQr6~1Ocu0#(7cG<*YCl?*XH+W0w z2+UIO6SXN2_Cy%tG*M(h;N>bK>54T>S*nP+Q-*mwW|4y|Iu)QSM^HaKZ3t(qU}S|z z-6DjKODV=&yFh==z@6|=nhgbbiq@ZjA+)|hYebuzCnu@Ghg8JPw^PqyeRGcjxSe1o1m*i zXycxswyax0NH~A5;0wExDI}f_ASBG>)5x~Ly^=qd9xPbqb0K2^kQwb*>nV=u^Y!XSB z0zd{r{)3m4U3e32>33Mwc(ETMIc!35Jenh=0RS)>HXAi3$;jq8I^1&n{I~(!qnk>X zjidWbTF%F(L-+x{3n1LQ4hNUY0l+7nlT2wl%PjlPygM z`>-qhl;ehptte~avm#$HxF3->!x=%)i7>q~cn~Hhm6M2eA(C2XT6#9pUDNOFCheSRDedlu@N)-brSsKRl4NX%PEu4CMZpanKv zbS3SH_{&)b38<>nQmqDOdI>sTscJ&m3Pcr@{LMU|1Bye?jgr<{BmFK0?a3d2;{+`w zfsnGtFw+(M^td4$&>mH_h?`P9$3sMAL22~TZ}e~#zy>mHBJAA(!Y3^!(2_8)N#Ehm zM|%wfjl6WVaBkMj68|RYim#L*=}Kukl!1XSGIM&>a}zDCXos51awb;bC?WRGh^6Lu$>{`2DoF3(>km@yK@iY9{o5TVv7d= zpEXq`cxfQj!w>=vU>m1E;SCqb#|DBZw9hBbPYq4+DsF#7Cji(gm$3QqQk@8J!hk5N z8#j*|z(uq=0vOpIlJ6fPJnuTWj`j-(hrnzPx|;p2>!h4g+92~w2~K#@=%9o@%jrKs zY)H7L%7tkToxh-87F=Qy!lI36pWv9>(C>pVChWjz?o8qNPmcp)WbdV0hW_hJh`}*; z2=V-q%lS3DUIlUTBjwYBxdVt#ZaHX*Gp0l9UW152(=mZUCxQd2c{9fl(-0s7z7%;7 zfqe{P2$3zEl@vo%MAf0oxy6r98@7hnF>CoE%mcRm_<)$Vom`7;t|QlTG!s|Pm|{@u zXHOXBJsSGNs!j-y`v6-QOk}Vgs>pwni z_;M61vY93K!vo4T@$hu0^Olp>lwZQjRq)o7mFNk!?>VmSj9U(ChSH&h#OGm?dKr4K zC}HA6OsWDws4j*fcvpxoXG-l-mg8~^$&h5(n-QH|lx0k$VxucCXmi~+^yIG$W3+Lr;YL->a7QZiMyJMp|T zYfhJoGY0@SD#JrPmLdC1K}_LECB~b_4c~;Av1;+6(ht%2c6>U_yG>qwZsFHe_@+2= zJU;IAYwC#{BFPqvprW$`qklXcAWTDDS>)M^Q;P$NJRJFD#SQfSRDlXaI4H*xr0&c) zH~6xJS6V5%#|`0<*`iH^TcV1VKUTh|^KO$@m+N&17aKVq@AgVf^+dRka<`b4!#FqO ztpoRKWAy?Mv+KGbHN)H`{A47ISsmsYM#&urTMVLLH0f7oLK@X%!Xc*H$AK_f;1l^H zoiaQft7CZcUw@Dg@YXg$ICnr%RZqxA7%UhWIGBXIm+E@s^cQkoh46A@mGi)e^q8c? z%So`E1uv-dAF_)1m3|!)nI`6p{OM@}*N_&CWb1?+58ypkkf`&1lb7xQhl^^RKy@T| zj#!&(IJw*q&>9&D(PCX7n!*DDxf;Oh(b!(t%I-N=l*J!uPzE8s%sE6;M^9GCqmnNH zhz@@Ns&O|@8@@b%MStour6XJP$N2sLe<<)Z_4B|XO$Yd%TyWm=bm-v#m%}VVg$Kcp8F@YX} zxYhSRJ#O$)F|u;;g0derIA&w0@dt*!fp38@H}Q?B?D0KQJL}FS@b!@Oz+j;%7zCrDhkxRR~*Ueg}hSN37!M=Ii046b;lfVsw9GqF<7~%Zar|JO0*>ulkYX< z?#Hd+Zi{HIOgtqZu!zcJK^nMGUlISsiY_=!0~#zj!;Q>NbXzM=W?vOWL!iwY*v6hQ zaDMx^0bKCon-{>90Ec64zq;T;N$DlHs{k(QyT(vfp5aMCN1E^wyIQwDC!hM)PH{507hxE!%XeEBz)cR5wRdo zxr`|r__i`uaN5&m)#FB!4~|4*8iH%KfzW!S)G8KMq82+k+&$@xi&~|Kk<+-Gm}oE| zM?ZqPi-r4vTDRiw%xU*H;H}ItWuCxIVVoUP|JQk+$xBF2SHX*#LRK8NPw~`qjcuL| z8JSd?x57Fubilj=SXwMOt2NfwtJP@|3{AYqk?RST_p$Q2%jCf`dNZDj4Rc$LeIJNkG~-_B z^eQ0EB#q%Yi&+qoL4X+%`1-6BTo}lK7??7*RX&-kpC31Xd8-bYcaZ!5U^_k}=B*~r zuc=%GaNGEK&`G!F=V|WjnqvbErT{2^7^r6fK3ktnWA0^8B4KC(a`x;P1HwoQi_AN^ zjZM4>UoSujXJVE9!^4Ix2*Zk}o_8Qg;p1(j_6BWpKS0?ZJ!gIT`CE{`-E(%eb(=}$ z?3-xLG4YPIWw_7o!XV@bR2+eO3|gd`$munN>i7k)n;~nN*w+rK#7Ge}RDGz=)!rW9 zLa;+h!wFRVVb^f#Cpp@kHpby^iF6` zmU+-r#VJcwb3Y&^5GweMl45Q1?!Z)MVggKL^2WmuR`JrCkPNO##twLn*Foqa45iQ^ zX%RGn(6X`CM_;5tUduB|grCzr7-Khuv5yn)mB#|1cDeM$!2!b9NO}Yf;wepnc)T-A zp|1S(z1u#nqxRJ+kIk0#nae*TZ5)qM9PHZ zA*M;#ToG)Zb{R)pn*z6BRQZLE%YcLAS0wMi5pXA$1OH;LfpV7ehAR@sNGPMZMsl?d zoX$Aku;NNktE?5~`kIgnMplcl;`jr~DRT46q2c2)ai#c)|t-{Z0wm(i1U1 z)?C!}5C?Q};&{ff(WRx$Q)WoEQ89oyg0gXc{sme z4&3~5DEPPxI1mkhrzs!dc6K*d9%8QaM||0E6Kq;%>hkI`!0=6f1+(?p?Ca1)SYd>qMyk)JsYShJa8Yp!h*ddn>>`0D9<|;>>G`5KfyLozP z-eK|EK1FVROE`R7hFsf_iwBGD?2?Fav_{c0ac%={Fm`zfPO)LlGluXS_K1?m zKut5Ch9|Y&$XkLEF^;Mbrs5ocn`h?c4H3T`Qkqu**Te51;11>vTRXcYa8Xh|fTI>L z!kca{Sc9cLl16T>$TKMpiwV@!Saj$|1LaPZ6b7KPvF4~kdJ5cpxICT0bD<_ z_7%u>c1M_&9EP$0TpqZYaFe3QaOgNm=-BCKWXmB{5oH^z8}x~|T}RZ)JXF|R;dy3m z!Ii=PT?Jh58*sJ*E;@I1L3jZNfg6rU$S*0mY%=hKPGj$Flknw&(|QY3D`(0#v&%Or zXmhMJVhnK{auI$n*f}vI^>MEh=o(&=HwR7__z$X)&_8K*LT~0tCIpmY+7=TTeWU` zRpU;Z{7)F}T$MYQ=%%5>_MqOztneQ_GnujgW65D}IY z%EdWu^V{JUJ}$$pI^h=b5pE~9LoC9rX!;h-?7~e%xO}V>li`vZirXoY6V~oUqXa%T z(OqV=&9U(6Sbel1qPYZaK^6GgUfWf`C3M0NINQ;_M`tHFrbro*i24nI3&S4?RSd#f za0sXZ37E0qBGSQ5W3`~m#w;FPIZSr^3CL-|dF*Hl@dUU9fkb<)uk|wE@&N9jFR=es zXN>Miz$N)3)%yXQ7jQb{%rb!!Zs;h{xGY#BIKs$EA({(;8>n3g96ODN3mp9%xdpr5 zYq<`&3~qEH7mtuTyB(tTDyjt-h8>BLBBloybg}y3%Z*G)U~6#JRt?=_%Y*hI_1G<9 zwob@JYDIb>XH(?Dg0&AHmmwGZK+f$N*86XD5tT$k zX>3nC^h5R06IhZS1G~OLy_B+y>hGXjZp1FI?H43d#BOt4PN17#5eXkxK}V__LT3li zot3DgCQLj=9>9q+kaM~rOtC)e{y!F@B-#2Qzs{a+F1a)saUA{SG_Hs| z2W~-X_v$YMt`a!>eY@tb{#%_~5ka%~Icmv;(dr8IjDeczjPuIMIZ6a`Q-g#mCN_I9 zOX3=56-FbD!}Z`sDtHCkf~yW6SHaf6VM5sK0JeN`J@D@Wp*XM3Jdz|?GFvTsA(ce>DNGUII*p!7Ei}BWZT@eW!?s|R!^btSq3>pc zt$Raz*v@W+AVQKLS0s%_k~|h$9gA%^ai*TbhH3-3*~ztKf&EItPP7|p!NCltSDj~# z-GXX>+04tZiyLx5&jy=^m1B=0mHoV7I>;bUE4$S_a%cB~*Q#^!Mbb!-&s%R7IHg}^nm&4e-I_ZKgiGt*BkLnYlL!k_HMj!`TAT%tF zHHXk_^h39R})UaC?*cYQ|K0aYWTPeIt&{` z%^@5>cXCCrcL7{4m zoI{O2DnJ8rxJk-zhSM||?B~_#j$3&tmm!x2aRp^e+MjPBqCa`XCh$1T02 z%aDr$xzL^d9dc*&X)`T4C%+_YNfMVNTds32vR|oUu9r+XAiN0V=PZ;P1G9y30P#;? z=&N4H;W#X&;gOW)$jz^a^b%jbsnlpKHNyGbr;ejj4sUz=%x-=tKPgSK7$C3*} z%8}6JIdpjmom~Z8-hzG$`2f0;E5cWi=O_6kmNw{gM>sv&@aV>Pt3j2S6@D?Q0Xj$M z92JukMKIdfhU6B&Eu;zT#lQH=fQth--wuE~t5IvZ3vIo;KS{All3zxyb!0bfICOc? z9zdxb8VzbbEE|h6Pdf@BY0ZIawO`Y!d8IOkZhl4h7x&^WgRWKu{oSqy(4AZn^u#zf zL$^c?Ab?KD>6r7wq02hv0HjkYWi-)69%qr&3gqJjoN}PTtf1!}H)608$W;ohr##jimS)PQ z*v%`{9l6)%%hJLHrF3BD$`N*FS47k~=jE38k&{cQ2|F!8uTD;v6}fWKQnfDB`Uf$W zup{kfva#%#1l^qBHpec{so|bqo-fM__uqD47j`Oy{kJ;1BZAf)SUG`R+!(Lum((IC z-EirOjyflbEOSG4Nf@aW25L5PHI6vPUV3q%gp?V#=`KnLVTU&Q=&o!6*4Y){XZ zCDd!(fJ;Zfom~-Lz!kYAexyrMWV)noWD+{*20@MkU87`BMqwfj1#KqxsH{lf5^mRR zP%is9bae^cWf}Db&!5owxX(15)v1M*6QO%h|0?;IW3n6f7 zLL}GX<6s)Xff(gNLsXaKIcmkBaQ54j+zPn)1@Y^-JztX+uI|xc_>{nf1K>{X2Up|J zm(+e7d)4HYB+>qj4kxi2qJ<@r_)wb_(fC-0E|^ z47ubgkt;3iBmT2$wQftt+}eXi1(gSuTN2aYei3wo2qBU)GHAC#oB)IWfzSasb>jpU z$->AA=Xru|UaRh~mFIF9aBTxF9RYX3(5HwBq?w|U+j1pdyH%UDlKru;xu0s1dCXC> zITI}t0SUU8$%P7(gd76bFg)QL=g7_Phw_}SL(Ze#Oys=nkvl6^yCme;=ZZ2VKj92%!$4Q|{daJ>JF}!i>WvMdus{{`<0&Bb?_M zx_P;p$fc{0Gru7htnQIJ8Kh?EzEKa8#Av!AVv|q8wrD6LkU5SCqgK&M9W+!e$x;ye zdk)BQx4O}j!%ELKD3|As@Vr>vVT;f4GT>Nn1>?2tQzd7$YG_Y9Aoh)7)RjvTtF8x< zlniB#ARQ%gAiyWl+)jO@gk$Aa^=zeXqa!D>&{I!gn_my{IXs8UVB;(>b*lHvt)1Nt z;=In0Y!J2i0h=xdKb9MZXsCCD^@e#7j@#z{);VtTd%-`u>u`f2iGT|}?}1x}4qo%*4QfmT zZsbQ|rY}Mf0rIKoeADsP4TSYJR-aQI4q+|Fh{?361s8Cm`r*Y*GfOrvP(S;p-_Pzc z;L-rD`{=s_iLDFNSU?2Mw;&nxQIJ+?zcj75n90h=GajQ57)ri>V zZc|@QL7U$S?&4G`qh2`QtE8icjLtd>DfUq~nF1sWxf zg-4)zjuErs2if&lYPd_c!MGJ{(@Wv$H>UNkf-P;gLhx?aMq=yPX-H<0^|#3hDWi*o z(LQh++Ux=8^xL{Z_+_;05U>ARtDc;3d#!mF|`lB&8N}O6f+ZVP!fC0Z!D3Hp{pN+SBPik zuuX4-4%+k6^Hs2ATx*0a9Klvk78}`D3kBLAM}nL^O2AF9iEC85t%*TdllTbC8UtK* z#)iRHC)99{I=Q=LnntpDF}g$6|92T^X@KTy+k+O)ZUfPB%l+7B(Lv1x)N&dWkv!PM zo76PL719T}x z2`LITcL3YE67ALAP&w5B*hV5m#hEfP<>qv{1)XlJ-@B0uuB6q(6Ix-((KotnFqZi? z54XT=UWM+c)eCVe12>mx2B$viASBIioVfioBwe>LM+Ci7 zvbkctfd`T6O|Zcy-sa&}u;nFemu1O&NIhV)9g}zetxoO)4slc_6dyMBXmnrdR)}(T zy&N}rxGj)#S|n5j>fc4%w1zq+6SdwvYOPKFZkgsQZC-xvsFkP7QoS{0;D({U1KiTd zwZH(r4mARfWf3+xAwlU%5)%kBBjI!?TiaVDH#|VgV?a#Z4sg}prr=7NgEqes%2R&2 zEXi9NN0QK1zb7-WbaExsM0M;Y46=cEXUihD+rt``y@y{2@|S_RjVp8X>^GM2~ri4 z^XBNhrK&85$4o;?Oh_UrRM`rGNjf)>GbJn4{k8eLbL8fQ=Z;%^x-7}tO3p;C<|E|J zu7w~$c6lMm269zih}n=)BR6u`i&r@%V{eB;bp|J;#|q>_Nc zQ#oU_I*lW42<4rCHZMAN(Ck9c(g3a01JKIZwIK1CNcnV7MhY~I<#S{ls5uem>x3g& za)0NJ8|bD$!j_5K$R4@TTBEM7k>xG(Y~{S@+}E0avde&L6nKEwa0J}h)!+zRkRKAq zbSk=a0oXPUNmr00l0Cey3tv~-lc8&t;{n>^O|#CZ~n<&1)L8X zaBhFT)svfHsKJ^TyJEC~n;a3LMsiy5EmK}LvFS)9<;sy>jHbJ++{$RHaqL@ac81!# z+}ts9Pwq0*%0MmWypQVZ$+f_gWFf~fU>m5ZO*#+iQ4m3nU>&$|xUvUll0^a$Rv~Uy zg`2M0;BcK59dGkZIxjUpIVdJwg_=QNk*MuiysVzw2t}$hPPvdU`tXOyHk~-93!(m*ZMv}hUAY>jv$Yd4R`6L2Uq9m%6Xx=qxSOa zt5;JA4zLQ{EV9Bj=CG*NXM)Un^@ z-0D2*Hm@>w(4K!?l;ce#?2X_`ImnEM6RB~WO;|G`>hkxAN%9Y$Qy2UZD#hr7!3w(X z?(N8SpjmFE-rIltLdmCgdtHV}7wF`aE<*>Q9MGxWA1s;rMR?SS(6<>*BY*p_0TiGo4`F~u13V+) zQw5JJOL*qh;NOtmh*!Zgr0$TQ+NCVQx)>~ZFruRzMGYO09R2OXh7Q1~EHq*9BpaM? z26XzBzeXo6(1|HrhE68gxIr*-k4`>C$B2&7tRAOKD*pCipo82h`q6+-KYc--JjmaK zk6Ie%_$=htYWG1d!zZ#VZbyCZ!A_53EgpK~IRX%MG`@Ylfs+9aX9ZAW13$7K$<#n} zL_|DA$5sckC3hJ*@+aAfv_E<*ehnTLH&S<#(pFh}f7k%(i-cNNj?}u6BO*Xi0G;5o zI+}g_eHA_#qvgP-9h}T|dNK!{%v>^1L>Ss7A0IY+?iqsMyTCUAuH(Vk#(N!-n=5=4 zQp>5j_<9*W%De4I@gX8+_XyDJ_F=$Nu)i7(aYs=m6fK0037_EvO!1kY z&Z+#4FPGsH+5jl!z2_57kLM5oka}ACAqh|{+&*jwrF*1I@LkYvg+AZdU_A^0LTP<} zEwqzUd5xF3%cCWNl0AEbqWdC3yxy(AAM_eadVkmu@<6C^kNIwv`Z%_j6F^~ofh{zW zQ+bI`R{@kz;v+z<0Ob>e7UGrV zSFD#Iq#-|Wpd2BTzm8A@dv7Ff>c`IqK0T5#PRo;o5i6*TW7@jM1c2iDB3q~!_X97M zs{n#}mH?`54^TcmrcK8XwE`Ph;nA3T`>;Wj0a0sPcv3l@wh_evB3s{N3%O!fO1vz| z5gDeUd1v2$8}jKnZH&aIWM5-<8pL1u@nHi9dkixw_^xL6jrBV5!D$V-brxt|&wTw| za=a|h0XJRW1E7ezJ{mV!;u`YfobIPmL|`34WpAkL^bB!~b{9u#hbXYDD@ z3z4tSrTdYWp_7svAz$~(LZP0X(X3Y_U0f;QSdOXOK5Y2dJ<|5P#xfA`+Y#-Ek6&M6 z^P1!9Q?UyHvL-*sT-figdU`~A9?^wL!4Z@@)Bw797yt#_^JLILU93Z5+Y#MAuWg-Z zofj8he|2~+OO*vs`VtW>#4Vwnp3hDf7@Z)L@FV4=ZXPy(A^?iqU^M`f;Ok(e;}S~i zBWNM;TZ;>q<;yhD+X3os+#aCQ^V#VF10>eN^~k)=&BKNdV)+abLv(Nz;4}PaUvOO4 zu)e?+I=&_Uz7QXaa)EllxLZN-ygKNZaLsgqK<$Xh<$C)t5Hj5GC2)&@3Amx)KRD*N zf<{$nf>2!O_2&E84{l)adr<#K7Kl|@wnk3fdXtQuu0nq9f(cLYi#Bx zbj;7;vV1FIFJY7S9pLe_U>F)p8X~mAh!7YQZa&}efeTFyV=@Z@pY4P;JQYi8eC8)~ z$WPbd<64pv6zX1kBAyluH++Jmbjy8sl%USr=NmrQTMgI0=Ll$HgM$d(h>u&wsBs~g zYx!5WEFJA~lMe*a{5?ML6dwi|S!#$qP!HfX`S|&U4{mtU$#@C~x3`KRIM>0my1u^_ z!nnTw372J~heQ*&2>0!v@w8-!q#*F2kXpY9!_eY*`+UITu$o-p_A(wnc^u7P*J;Q! zF5y|I-@05cghxH?0Zo|u<#SpwL?@3^U&oG|IQ^`l&nX}*yI2m(QU+Ven1l-5*((8GdYSg{rzDB2rJ4PZ!olBLp`Vt>NNu7 z%Nn3LQ*d{HLcI(iFpd+Tj?@mI(~4maWQmYuS_<|NgpY3?git?_fe`LGPij#}Q;j8* z3s3Cz9X2l+zP#iV%4GVTTS~Jwt0$QS}fC0DJO1OR45CUSxbDB{Fc)m2dQ2@ZD5V2@PAi6?GE*iJ2h$q4{d_|Qk4@-Q&(z`AU`%?Xa01W# zjQ*8FNSDErO-@iM@c^FFGrH;)6K8Zkk1z`!ZXX6f6@Y>12xLfs2yKV7aRIGM&T(EZ z>N14FsD|}3a(@J!)(g>P4}?T+1Plg;_WQ#I5UzQM6+KYI0~CZCK8RQA`VO1d z3SV0G7s3N47Vz{q=>VS76S^?U*hpm)UFCTDu;GJSJ{s?^Vk)3+M>LjG%)BFc3Lb993vo`I9iJsY+1a07zhkDk51Y6SY zV3)z8krwD;l>GuaEfsF?i02fUj1)h9xD6obrb<6z?wp@qIudb$uH zF*5})X6(1wX~A$qNYqSB_^>wr@nJ&7dh+VaV*83LHmI0f$c#IraO)1`*PW#{14`5vgPdW~RsjVq7D%5KKzH?Bz0q zKm|pFs;hg1@^=t|)~})M{`TP`LWQdhIcP7{1|r1#khS$)zmQk$(XKsTg-|L*M|E|3 z!r$c1k80j#0tNl(Mlqcrbj)8xU}#M6-XT)9^vu4r|X&kF-W;p>xz-x$6+J(ZJdL>Vrje zdR8Z%)nY%!a3zI6Zyz>njFY5@OS6lcijW>0 z=f6;9cV%w)>HXuK`|U0(7B|OSPDe8C)o{~V)0MPWXuOd4ZB?HIlMxk~UH3_Y1yHvp z$6C+^0>3eK#$2=|&Fh~R!jbzz%D(?W6l>V`U!;@zCo*~%F}9IVH(mk_%HKm5@AH%qJg}f7i9a30lHoO?!Tr0|9e#)ef=2g=R zQ7vd6Mb+J*47|UdPK&0V`9jEvpxFStLufYKG;#`sZcOZHVD0zVz@W}G&VExvr*-T( zl1ib!N-PN~#QV|i5p-BcBtPOv@<#6W%oh3Ab_;5%0^ZOxs z4zf2u21=|-#^M^<9Iv04)$Di$P)$TfPPpALeOfc)(m4UQoC?kqx}^p>vm6b88MKqC z$ibT@T!ms8=*xp{I+K>UH44ZlOi)8)E=k6!a?pAP*5(Z?BgQ6D6!RbZZndhL@ALvbjxOqaX)Ip1V zFYG`&t?GKo&qU9479z>g5>?7dlS*Uw>}F&pU_h;A~!x~qbtMM~vmFHOnX#Fn(Xf6q~ z#8D;VffmGe>e}4ZSd@RStKb1NlF2&<*5Lhtww7H5XjPz9R&}GufGcH>iqTao){y^* zrASpJ=x36yo@eI_)_LLi zN?6fCj-8A@$hmz+@}%$_ID?C3Lru;_xMnVk@^vuXAR6FM7*ixr+j9utT^Z765ssU# z3o{c5C-0T+6N31yFKoOpj~9Yf{GMC{clzlM`teg-^Ity+7afXMD1mrn>7-ok z(fNWsug*J10krXec_<{dkrZKe-!=YQy?g(9y;Dxkjeo-#l5Km?E4H>iz4`X~8I1Fi z_Jzw6d2Qy)`!cmX!D(7ov;!qtKm&`w9qO&5R7Vxz(-U3~RkScfz&ozK@$x}?7JQka za2TS&!HITv4psais7R02&#HO^RbE$>N)d@V$_X6k5H<+XFdkI&h;D#3MEUSlD7{oT zG65$n5|`C8163n-D*B!WY9?2*EW+7&#ryXwKCZ&eNb|r(_T2%P*GjDc#An1UgSN48 z5CqUvWPTf|F!-2RH_J9)C8vKi0p}7MGDajH?4$U8nbpo&fD-s)KQaE zOLK1hK1NY>LrwDCWv;-K7sR|z__zwTz=#8|`L=VR^I6fn_&f-~dG<5n05)1}RjUo< z1pv40;b3<|i_S2?1>ho2Ha!0cHH_MyD-OYeltM-;MY~y zMYDlj*o(|%JGmP2@IJif)ey$+%vI0{xkwJ9YnH}R}t9srg zEK<*4Tk!lgd|U<_7@q*!QMYfllbazI=@z!eQw{T`geLm;lBwpGAK3mdP^J6^FtNr~ zl>YxC%gxC0Wh$5s;HX&rIM++diz0Nz$5pr`R|anB2)C1)A?Km#NSrKaFS1X%QRo+_ zB|e3)9e8lypm*3oa~aunpCtrd5vk?8fhNJ$66bq#QHw74xC}Hk;~`gc09rV?7jl#+ zYstJ9{8&!8a&_fA7Taih8eU29aNi;%s;trrrkFKxGn{Yv{d99wq9D>!!xnJUH0B-eEvan}-~<;5LFGSNc#x*H@lgR8Llg7EMDPOij=y}TUlha;sKH3|%3 zMpXGE%#2^l^ZaL4Zswq|+EwM^7u-6Ah4vG?DEcB35Us^^BiWAS?!zp&?Hq)+MsN|S zCahTziZM%oO)#bA>@n4V!nl#YT$k*@j{V4u+#1AS5L=_i!kriD?NFiAtJ>;& z`H{BiH4@i{!H)T^t)KxFyKUY5O&UKJ$s{9s5o_PYTyMb7dKUVC_t=^~p{fGXXV<^O z{qNn3PdX{SoMM^%;84^=LZf_l#X?sY(T}F^5m9n}A$)R)NbM3w@g89`J{J+aT4ija z2;-`H#7p=h3TZ<(W4zOGG0t^yK}~98(<9A}O{W|o)!s*0X$~DZ3iBWzOW3Iv4F{!2 zJW(ZdNXUGz4vi*q>zLOs#;&23#t+n3MCJ(_(=2f$eStU zfL8a?|NPiKSzGimYH7pP__%o@b{j`bPE3L5x) z(1D;S5frqP=5-Fj@;&gT5nEN>sNCzw)Jhthmm3i0Nq5P8w)v|CMyNNAXin~t?19Es zY-6*v!D3pqU|85^f19qYayO}`TcxUE?NeYlFz!X{4?&OKBS23!5YunciMo*E$q`A|XFF`}T!fSF9RkK8C$&AA;KTJv05?z`*TR4>obye%usO;R&1hFz^y2JU;(_(rQBaO(+0RZ$zaZK+!jlH$`Qvu|WO z{?P}JW^17FIlaHU!p4wMK*_ExsXYJGD@%VM#8q+E_=MVlRhW!&YSzQmx)Rk3 znFB_#MFS~tvWo1sT& zE0p$aGW#g{zbj1ufA)SQ^?VBXmO~Ua-`lo}Cuu9GUck{B4k3-BI@MmHG(-aD1yk(X z&KesHC*Cr)i&4k7r&%g$n?t$f;X6JVTVM}V^3!U~q=Jf5ImRCI{-XGP->=Fi@&kz{ zH_UYZN@QOtTdsW9r)eAMk_7t?P&4K@M#GrXtDB;Dn3jRIx0JQ8>L@z%gywB$F{Qn3yULHFeM8=^XW$+&2 zdXI`^IhoGad7+MGU zo&UEZA}4+!D~UBIGWOXQCP=tjs^~DonhTheeRSqpO1ez%R}a8aM6N57@h8j?mvM;d zF}m6VEi#F!6ao@uLC?B}{VQ3n2Gf z)!DLFy&30At`qVD8e5Z`+&FZ`ml$6+NRc?M>>1d`-q4U@KqCAE(GpP;Pj;TCXp!Iw zj|!8@RdRJsO3?9u=!*7)Qi{gL#^HMJOeU!gz<1y3`6tA0jmyCiIZXK1h(~5*vjw3w zeOVQ!oUnNPr%;0^O_fT+eN&^AHPU`-by;R1%K2-_>u|;-55VV=oOOO`k8qDhX)5oi zsUh<6Af8<{)E~b&?J=QV<7jT6BuI@7HY_w@wK6nN zJ^mscD9YkfGW?#Y3qI`w^ZeRHfkHymku0q83&d5$CXyCw3@r4f`Owa_rFrch!(NJ) zW-qCt4C5tmbRhvI=y|U`8DCi$5dAMH>4U#T8^arjc6}q7QLRfZpSztJK)}OBAk-q1 zO9_=nDlOvrhy&3QsFHI)5rvE`%V1<8hkN5Kc@d95kX$-@clKw5=x4^R{QdrSkn^x^@UWfdO!ZUsoA5TiIn9uh) zYnF0~U{f8EaP{*iVu*;ayBa>l1GNb)jFKkt^e}x!`;7xHxZS^%v?slGITQx%gUISkv z{nw<gW_Qh zFp+fiDHYWjof8b@GAr%U5y~o85hVl<-r{6-MMD;)Q*6;_KUqSkx7h2-CzTMfKyP^PYJ3`*}W5I#w? zv4-lwk%>FiS_l;Hoya7BSTgNWzc8?ekPNpa>#~;jkzQybKJ~8tVTPUcCqoH4^$;yw zf(24C1@>vkgZWmr6Mx6<4s9$$Cx_8qu8N3wiJub|2mZM&4`cPNL_nmZd65Shl|ZA& z=Gv{p%gw48gnzE@U7^TPgDZ)%|0H`RO#1+J@sE81PC-FOeYY7B5xO&LZhR{VVDD4sIdB6F9jbS@p3ugfDAUGQ zcY75rMIo3?49Q2nnoR$Xu&e+?0E2g9B}b9k<%BozNJTC!IluRp*)5{*RecLW$eUvg zSJXH0h|gUnlsJ@=eYsNAdECLAp0;AiYj@HuvxD-3t++Me049iNPWE|{kIYe~?@iu+ z14@;I10?=SjD7zj!k_I&e>j&2q)qiQ@{bm!uA36a@lCOk;Oqvq%Q2C4Glqk*Z*F5) z<~C8@#AQktDJpF2TG~!R4`12tBP&lBm#C77s&yASXTtm9PjKVTuhT6{I&r$p_3GjS zwP;IrW_(K`KYOo4IiUqIBCF`YO7I8$d*~mJj8?w9&4JFo=A2;wBxC`?Rb@IE1s&~% zXo$8-%s;WNb4sBPG@jvzdB?H#`=a2(jZuV?M5U?pm+cA*YdM0?|sTf-=s zK}6DnsUgj0$l5(};F43?xO|_hf7LW@C|XuEaVnSYymb>ra|6XJAk5VS$Qd1C?}Mm| ziYp)oVs`gB+~SSh7yKZMMYJ!8^)`!a_eN{cd8S$If>|;N&%qYGk`Ah4+HA+;l-%-*JtW1aP80WsZH%=aOojQ&QUSP{pCQlP?7U+` z`8NIiSZ3Pp-ZsZFR6gSRbArakj+<001nA8_vc-a}71hX}J|`G)I`ka>GQ)0XCVkSu z`ADg9_$a7LGWlPwQuq+BuV~h-aSs7pQdfyQ2((-b%WX?pU6&SnD6;%Yo>5#N`R803 z^TDpKK?}8>o)H#}5mY-6WPG`1$sueLegb6%m4h&Hb<$h0n|;um;bG}FNj|p3bZ}fB z&h&#EsZfG9F@Jqyl4LjQj5Vx+A!GE)0SX-Vzb>D9gTQL62C;LB{Xj|*U@@KZKi$8m zZt+T8JIhnO+uOg5V{4YmZF%QLLAe~IZQ)8;#StcTWio+y)JBaQoKg+^zWk%dF&yLr zFq2>|T3M1sMLUbIoTZuOw4iEw^*zqz@qhKcbY12_kCOYD3H`+Z?58dp3WQOWmRKl} zk|tWRg$aoMORs4i-{lpZ^BMl)AwC`+gGs&L(2f$>)g8K8>ij&G@xegQC$|Skcn&%W z4_}mKZ0o{0KLmt_!3Px*lx4DZmUW-h#Mta%0+RWR$4)F+AS|;zg?XL}nQBT&fZMcNA?jA5PfL z$=^Y+4hq@29h$kpO2vGn&Lpzcmgv3Cro$?71f%@96h?nsV`k?CdKs@nE&-&?1mY2D zPDDSYOUFMS);`xsYX)TU5(yBH17LF8ck-$V6DsRdZ1x<7S4lT)xlaKsm%Fcm=i+a0 z`$tKCG*4$edF&R`5Mk#WLgykIW&1(a?vBe;@1Rlz!zCcn+q%Gf6XtdyOyBNjcGZZM zngTWrfDIV7<;041^*L}xi&4C>8>v@nKJR~UjpM9dqf;1$VA7=6!d|KKt0I z3=(#RMmPTBnfKP8Z3fkd2K?+~WLrn%!tX*rC%%oGLvDhBZv8fAvIv4QX~RjQg@sV6M7?jI<1pAsHXt$;fy(Haz$Oh&aE$GyQTjOy*wGl~ zOk07axhEQ!4u+c;vAtHH10v+-KzCAb_RAa3J&%Bq_4z=Re-+kY^6xGo@Yg~9E_Lpk zp%tqZm@$Ef<4Kdr!=Niwy1N(*Zf&Y%#5@$=9lcwjNUeILz&>BZ>M*(*;Pm|?l`A2y z=2dy#X4Z|`6FoW&p9+N_@XaqwtUvaQl+Rw|5nq>gV#1EiJ>H*uLThfMLEj=J7?j{W z?9AQ=>-KbGu_)y(;LCLI)Jy&X%#D#u8P)#^sAqHefVXrD4L-T(6t=YZnxV$^D)-g0 zF*V|-!?%^bdxhR=K0;V*Yq&Dc%5*^)MmHA96Kli15Gr~w#&RlGE@qD*)D(NoYyx8Y> zS~ckbbSGApT1^oiAM|p*{tS}ddSK6p08L9W&6=pX;Md{4TLPA~VSIxL)xOOg zz3n;?3{%D>ZWYtXZTEla{fQqW5X>LjwBlQ{=r$Se$)R`W9}=pEf0%S`Qwza~QKw$_ z$pCo)XY#OQiuw5Ai0*EfDasUrHT_RhSX^4xE{+^O?+CO zIp>IkWs(k@I+BrjF=95H(^%8)J9i`OE)&WGw9^pw5mNN(ee-(v2+Vu*`MePUlul#x z)0ZIUxG@Gh{=kJIosK#J3U(6t{%MH=X6{D$NG4Jc2oNk*apNx4M&0%ZR6xyNROA-o ze$ZW(Sjp0<%rN(eM*`;rObhwNp&s5%i0*RrL!@q6ZyPqDzLf07GtrAA@}d5}D?aM^ z*IJ(Rcq_Toa4sxJkiNcoz`l&vDr=%KE1ifUIqFsZ)cxu5t>Sk%yFXz3<7zaC`_F9| z4cfwG*VS=HcOBJ_77vIscNck?1*o51s`=DrF}dmI+}7#705TQ*gqrGHza(Eps(JK{#^^*K(d|cKcafd+pfOBlvvx~?wMf<*Rp;M*p*Z&( zBsH33u|eSoCsiKjoOV1F?CKA#7fO&U$=>Bzg_?{XjJr6VTpiL~_WrK*)&>hWR zN1m@?#V^K%yZGVOKnm2_ZnQ#JutLSSr5d{e(U29&r3 zHb<5gi@dq^+zZ}UrFEk-!wwy?GfFHzGYIObvys>D^5SyBRTk${>gf8W-`NL{-&E#=IO{k)9G;qS z%2D{RI5cJ(2u3?g10x$^p8k2>)Od9957o>{(_i7!2?LK z@`UH&2`wQcqw1+oLohGt9(+A`%3WK8gx4G1CTo1gePoXLJp|~-MJ(;6%6Sb~ni=>b zPe^FTM%BZ0Ri8L{p3?CYy_I8kHt}5)+94eP@`53mxFqhVpv0w(|2OS z&_?LtxKFY;JwQ;63b09x4coM^@;Mt&QdG-lX}UsWItufZwB`x7#*BVx4z*kP5Z3K{ zZ}lE*RXfy}`vJ!3p(+hWNt zjyTzid_OOMhZ6rfwKaskG=#Ig4zM;)?d42ttHVW-!ool*ZH}_ogngzfGJu}6x$;(T z$~Yn1MCILICaM?`*QPP}MY<46Q*Nj*%|Q)b>r3p@QiO%XFt84^eVok%u>jd9e>^k@ zYidy6xin8flbaHbQ?Ee}Tg1@QI8))%_2rF#AcQd6QQ1N}uZ1q-4k68VPGcNwnR6X* zwEU=KcUTP4&L_}kBKK}!buo0jh8RzryK>qX<2V8lr2gF5?hnbRiuny1>&#&v`&?k`CY-p!Jh{>;yIbwY9J93dnR@2T1ICi z&n(q`%okv=7;}Dd(5hg0cucuxZDwg=poBk~^JtqYcdhKoWx?(lt@Yv14Jmb|xS4L9 z{V+Rf8$9cR&{_WTswy7~Em&J{{{ zFgY!l?90;?O0)7oyd+5&S87s~PXC%;+4@T`ic_6Yw(Qqyc>=c^$yu3RH}w()OPJwm z|E>-aDvR@r#HRsZ82oanw6wO%@YT>Je#+f@a(i%WeaEJ(Uk_X+_pFiT-UD?{!15BP3Ci z+dwEf7o{a)wa5~)?3Q|o(hyDcVW%8^N~~gXcyTV6`_oRPC!>OsohxfqTj?Na75 z3oZBeTAq~hZ~Egfs?Z90%JuDZiH!=4SW^#EQ{#wTOV;E2jTOeSsyP9}Ke_zuA6a;s zNs;ej1H%b&oLqvmmBn_21Ul@QJqKffISrl$btlMN;Yae=q(V3I zQn9A&5CGI0P?3izShzNXL9x#qFx?jQ)OKV1qZy3V;V0ixP)mSih@DP&5CHMex7V_a zz%bRt-Q=gHF{K=z@A?-qGa0S(EShMsN>aw;m8Y-cNetNY7aTd@2Kwe-va`#E`p>4; zxn&!gFu;FD2-T_8P~?}{O16=DSXQb@`U1oATVH%41KCUpGr2gjEYRL+RIg7aVH(&Z zF3LoH(Cdf@OVUl$2Kw`U_gDoC2p^uH24d)_&auE?cbs(<;R;+E2j&sPFZ>M4((d~5 z6g124Ex)eeV@5>`u>n}t)DI>-28AW}Qz_S{Kq%dN|Chx39c;0yI)n}_>Zb81p0g~v z6~4LMtT0GhW%{K%Y)b$O*Zois`|rZ(pe6>pQhoeCZsv&K&^17LS877=Byu|iqae2Npon`aPnre#YL`~QFA@z9oR^0IkUHzq;;ggl>nUtu)7wdh!L`)! zgVqqx+H}lp&&56mXKYExqV7lQ`mxp_-9_yQgBbV=0v8FG*(c7x_l{DIq>#=rr0bd6 z4m+Yc<8FcwC6$*i)B^?@9)FEtuQ891{o{igtKvO82kEs4DNXCmvK_&|vL zN5%3|pj@ohs}~aXMowsdWw@WQlUyqciRG)q_|T8 z$ERTOx!hz6 zgJ_#VGg-Hf#1gR1a#b=A#)3`C)8$OMKz|LFhIEEc zUivbn{k?TJ{@s}K7ZE6eYRn2LnTTozaG`02rH9S_gOY#5zn&kyM5GM^P|TtDR?F*c zY;@Edd*#m}orQ=`)ZM-Y$V4ao*pIp(LqSB?TwP*OZYG&s&9lT0br$}duD7W;nRurKrGM6|P!)+c(||%3 znUh}%0;11%k1?4sq;sF-hT`gStFmk#FKXH{c3?v-wP71P?(q)pjUJ4=qzHegYipz> zniSWO!JpNapyd)_ZPTKSPw8hGAlTJRGHBQ}InGe(ifYKuoB;kI;;sgFmgmL7ezS(e zcuzDN*b1#0mLtINP~xG)LiY`#AJ&lVGyE`WVK+x=(gWo30Oq6T%ZOV+w@LWv$(rRj zM`fjEpWu4ktbf6l%E`Z`YMK>w`ne6p6rMUxpQh`2oqC@c?r-sB)6PhJVn5ZuGp2N) zIxbE#EnfBh#VlE8jNGvUhYT`^$kcY~|7Y zlWh~*qjef*o#>3W?ZtxkCHwybSwFIU11v2;BNGKY@x4n5AN~)`8I>-AIL(s4)TQzI z<&9fE_udiQIvA+r8AZ%{Eg5fC;Pgyzg2@ATIyB6uF0#Tepn1By*~EFjqD${#rX|@9 z!=rCS?yL~3>Ft?~G%4tqJUKk_M^~De&g%Tt zd}PZ`R^Derpze39eGBVs5wwPGkmsN-*ri1Cr$H9|Izhb;MAe%LC)xbgH_szu)xxqw zp{LH0Jt+b7`n70h2)&wO4SpMQDr3QV`+O!>+B1|wE7?J7lwwv}qWc}lja~S_HgUw) z>hPG00Jo&*D4Q=#z5DFqDkYy~yM`6ys|rt-X8l@Hny49^1zKxY)bbPj+_%Ls5!nfV zG)3ER|C(r%Ocz&?A}eIC<@;{{)m_2O{trrkmRl7au)dr$=)<*N?XsQhA{GSkStWHhc_|*k{ERWbCWkN*FV> zh^R~xFD^Fkkec*V_WA_nJmelp*z)0d%v+!jRt8FKPNoj%HtyN`JzlmI>@DVDj=QDm znX6+{O@gSFAIlXcA%^|etbmi73MF8)K+!0`V6dv^3@t{5p#+WU8b`ueZ6Y8Dir?FS zR@|gId%UZcZXJnhuvbc%T#Jm``hS`|A`%K~Ib-@0_yXVRs>x~VmP1`z0W2@PJUKn# zf7K?|d-yPr$+^sMt^!LcQ!SQs`znWRoHuZuc1;usHnx){?h0>apBPk1%`~UJ~CJ>01bbCL$glkw= zerg=@?@I!aqGJDG`mRf;pEFH(TBWegH?JbVs72J0pyh>)%x;M=hnt$&DYZ9A`FVHf zb(lEu`Bw<2Er@?QI=l4$W2O&ZnW|J zEW{bJUOPn2Jkwn*QQzFiD`^ zIi6)C&BmVdG?@p7=SG-Pt4ZE;pyrJ-fGd<-E(32p&~SA$BlDpqpX-VtWkUTVRW}Wc zp4UR(>sT)T0u5&zK+y$G+Q|-6jJ!Dfk?fP(t+}{AH@;(3(tmsZw3@%Np7q^Jiz50-af@8TxRLgHoHWrqUeM{l&{3(u+Bt z;0e~ZlRr%9(r`8So@;RKMbD}LN2-#Y&s-!I=hlp)hQNlT$1)ntU!CjVwP=2|9?v}j z)L={HWm%qr5c=t?2gkvr&IPgnJ3z$0IrE}j**=dibUbt=*6La0_GJ(=Hf3$IfCqN} ziOBuH0;C;DP@=_`BP1|waGOL5btNt$Mm13Zdv%Hrr{D>;7?RJKXa@Rsm6yfVbe$`; z9EV!sZ!1%B>8{0XYP1FQRf>1dtUoY9l)clJ>k*`%(0|A8VrpQmZ9jOf;^$i|+H~gr zb~&!b0OGACh`wmC9f1yusZ%M^es5~Ajc1rD_WKYo`@^XF;I`40mOv2h@FZOIJlE2n z2P*X6VaWNduLQOdJ)>%ANR-TvO1T9II9aXt*4~!F1RjeA(Mbqdy?;l2n}sd}ou^L8 zR^cc(FH2UF-gvk{tB5NWLzzszA94GyFefjT`z8x?6aE6jYPo!U45jVB>85_nz~_#Z z<@%B7>r=w!+<5X0!-cT5L9_qgIl~3xd${qs1%D9p+=<YVMF3BIqGCYSt+4l7X&w z&+>}5C$KP36%j|UXk0oaW6*MSyEzsGmMY&M0%&R#(Za&9@i@RbE{Vka4*DG+_G*07 z(P+PTZ9-2NI>2+Aw)hHBiW7ytm=B*IUkY3eC0bsDuE+bY^;kwius}TeSEnbf3X=62 z6fY5oA^qn0qn$`qssu#3RxUC6c8gPtH~#?$$6o~r6{VivHi%zP+xJYvLc^4*oo_Yw zeEoQ%mLD=l5kmbj2M+VClKa4`TN*33xaolyJDo9tv8MoGuzduyG|!GKNOnr77cQ_&my&2vIQpOXj@@PFo$)icvxHjXL2z4!8{fWm8gEw zHuV+ z!8Y7Nrq?OA^wRt|QWb>h%iDEe$tG!J!T)KX$_EMQ*><@_3NQHAy=eYUC} zY>Kn0{0sk!1Zs4F-_zL#!2cwx_!te%?x*|4t zoU$QA&56#PNd_C{)EBwjetw2bW4blhZ!`WxIFOdZDr-69PbRIIbadT>L3QVpW#M}s zgDLTx)jU+m+r7vZa>Vf@s&lxh6m^>WPpP$F8~;nrEK*7rp4~#&rbsGk4&>vwuEH%` zy^;1r&OuI=a$}2uKH_PHRat845rk6Dql%XWx|=hmWpJ?}*9O?Q_HM!{D`u90bbsW- zE6lg@ZIpq@ADUI)_%KA%inZ>upJ>~cIw20An9j?326htFA3?X)yt{6nGzJnE4=j9Z^GOvhRi-qq~kby zmH2$&6fRq|@kNF(g%f(bJnCp-B=t5C5$ev)Py_Kl5fZ%!95TCFUj(mcWfA@`zkW7M z>6;TQJs`Wex^t+kwC0Mef8~kBE^07cY;b>91WIc)N7+c#`L8GiLnk{sCpnj5yC9%s zDM?>0#^dDxes;d2)7|$5#W`d-DwWlUV4KBmgb$E3a9}>Td*64#gzH!%U6wp|2q#qx z@3-bgD&=g65S;i7)=vrj<}w9ix($>5WRI(=S5=ZWd&&qxIDELEm@$Byq`m+|`gruL zq`oiyKFotKC_$g!@I(y{`X=tl7KXhMMnYiYD^wzm;Wt~vKEx}=L0Ko8^xKN712xo` z#mK1oCk}&BHL}p_U)y`896YXd4UR@PzW~k2t$S4>vG1_4JZ4G{OAhxuPzay9=fq)C zs;P8-1L9TsO51KnpzibSO*YgdN%EsCz`*voJ8kZ&sA94B(*G1FAC20O&`bwFH9S*?c$-?<%C(3eX(}tK`DDvY%kXPF*grWe~-@zPh7QuLPoj%kZZ1u%&cIT2mh7 zX>#s#w=oX+Z9AfyissL58YL8zovvOT%;wq);b)F@-vzHSmtV?3B3U@blS}Yp?zQ@xl#|KPPYj*xFaN>ms1bgwgsRwS@@$~Wa$4B>#PH2bX0Ed(i6 z$W-2^zzw9-(4l!(7-HDl7C^;$Cwl|Fz_l}IeWb&KJ+5*1&OOnh{|0zrZJdszfwsmI z^7ac)R&v|g5XTrqUI9k*(;|D^UvE1ex>)nMyc%*Jj6=a5Ms|XiEgMLygIXjoCK=2k z19N3n)%u3YMZ}O#z7ub;g>p*i&L$=LQsOsPife_q))OK1Ws{r5jMn+%8fdJ=5@Xw| zxU@xy;?d$JV_F7*R+MZ3wo$bL8SFYF)hsi6KL-DJ_J@Cze0K*8^;>_6*w=+ovFVRj zPm3=QowWCaFieyV#5?5%1O>k&M@-pLI`1(~uq%lXyUe1Rxm$j>o}>vlJ=Ja#Pe}_Q zWgMW+Jr4zkInY<$G}fJ7uZmIhDw}>rK)o&|PN}E28u_Sz{zW;Oq27X0HE9-&y3!SB zx07XCV?W5P17daID)bk{YW?w|=qxzp0t`&Z6J~YZxE0HtBx~q>w4rY)G2n&*#aT;~ z%CVoRSJBlQ3U-+pHD6cd#=@7Oo(GWb=vX}Oe8w^NPOP2hB@h9P5m5`olL(sGbjmIq zMCR-Aej$l=InB*Wc524MESQX|q2Ee`bf=?mZcKg(isfX8w}%)@3)bCnA~7AVe;(Pu zh)T&_Lt+h0a6fnA|11i0gzxx6yzl)Er-*ADp$)OhF!+patLfPeWoI2>s$^J$$e|iugl_$+ySi(^)2P4U^(OTwo;t^k2w0oUS?8jEZiT5iqiz84sbCRBYRjhAF#9&Pn zURN`bsQI)boOk|3|A{Egqjk%2`{}jh@^HG0AZvgxPb<%+ft-s^VMv|(Y$3qo9|eQ) zpG_+)xPaMM*()}vf&7!WeBTc>+;UusK5Y^Dxbn1Dkvos#_Y`9O!bo3OAGJqjmObQR zE?*8?PlzG}FZ}k=W2EHH^xVh_#Pl&?e|Cig@1`tREw#&S>vC#w!s!XAH80gYp*fT+ zLr%Ts`eibbB#ztYvHct3!JrW#s)(vi3VY4SN|aYJI%npUM9QZ`j2bOm=*5=T zGTGt~Adh@|M~z`no1`|pn}D&PLdL%pHuI9DxJf+iKyqo8@)!*a@7Ojw%Xew#xVuTL z$(SuV+Q}!4`z%EJ;K?WLC5FdXQj!RP}oL zFqmbU(8?xy79GM>#q$wUYoKm-=es81So`{e3J*U|q)!L2>^Aov@xDck5WgJBADihT zVw-DcHH0@3t?3|Vvc|0UN*3&JY(&^pNNFU%wa1Bc6$U=-4D?U0jgI>4B?89#aQxCv z2y{iy5K=;8;7+xT_Dpf5u$0%AMTYLD7NROaHV;i|W_64FiK{`HmGdT1foRD`bJEWs z8);w9A5!P63I7&|Dog{IcVTEy>>MRLM#;yr?w2E`x^9&b`ha=T(XCS3`Bk+?&f0L- zPSUF_uKC8viKW*REr_8!C<}*+Z~14`$T(wQ^DX?7S_cmt_Wn&&EJxn*;&WJ^Tcx{& zbY$)&I6sf9bC4Y07w{@3+8`u7CY@`dX-gXxO=+?9V%CcywisYM4RV-P(zGCuLjRIo z{SDt6H!!3#AUx428f;Fnz(y5M9i-0r&yiJQ+gKYcpx~_rOD%oX1D(D@Ir@NTbtSnY6Dy zoFj3eoQR6?6p8=!7Ud>@0v@-PXYut^eCXaBA zU5YKB*)y%UE2@O+a6xNQa&yI9(P=pP08HO^rGrT!xFWW8!%J;Zd#w6`Mt7Hps=-E5 z2ipQpL_w+7giYm!b*Y}x{011*$96w6=b$xUPjB_5IFqq6?dY9pE z`>*ny+r_gxQzWJ{(!PDy1)jOm`EzJS)wOE-e^ET_ofu!g*G!;>6ozkvb<<0`nt4kS zBIk)kR%1!mDo|6D5U^FbC)29YRVJ|n(V!Y@-Lv+#6#dC&UihYf&%^e6wHW1%U3bb> z@)HAY_%gn0pW9zXpBIZ_)3THwEfH{bNINh`_d8H?<<{QvMJ$(5Ru{ec3tBHKK+;*^ z18D;N!)ar)b7Hy|SyKYXH@>rR-vn4=>;v_(m?=#!ow)307QURA*5Dx?Z}9d@6jbSO zY4bYEmVS?$E7v^uDpoGMxLo3`%@LyLC@KI?vK9KB60w#8bTWwpu?iKfXsnQd%RwprRd^K zGR((?EKA|lhf#7+Ol1)i>1D;nmWI`J0lqr>|MIhhA;p~u$k>2WHMsS=FWL^ST%uYX zf8UJtm*g3G&*Yh+i!PRf&7XI^UjJ9hf7AnVQ{zUCK#?qN{}&iV0;+7wAE1gK_Yk1EYlnA>)k5dD)(-A2kq0GW)S6w)31$DKpL6E(NinfLjMSy zP-aDu*>#J&Jpj^>o8+LO>u3tUjT=V*zYSRnkFdJap+PgP^LEl_FOFXpT|i_6xX1?r z+dapRQJ!_h5WaGDwNOl&3hz?FUo#@ zwme})k=HQ+T|XEUixqU_A*xiYE^MV%5|wr&KAk1VL>SiAbOTy-tVRrgc;G0i%<-S> zAg2m9ASB1i+1P+iV74;>S^{7#6kxuIh^+@_2v!l-Q93OrYh3m4+%CN*T{fENyBMY) zHSBe4Lm$E2ehHkwDzSs=#9TGyhHQ#6;M?Cz`wpierSCVJMa%{IQ1NfVprIG&9ILg! zkyRDR!7el2t@KXVUWH`3I~Jp3F~}>3fMULU{A^rj^JxEn(jf*Eqkrfyr@oHhb-(r% zcIjtumTb?}&uPvWPo-d7_m97$NX`Zk*#=U%09k7ISo>s3wRvQ8Xfl;q;NT|wuQNO* z1}q$+$GfKZUB*eR-%S#ZPZg99&0VU&gYm9^H7nkmW}r|(BoI7Uzs~?Gf%T=LuX@f< zp$92f2gChD&c?R_lp?kV$tCRfhzGRhrvLX*n`#Q)DMfCKcH6=n$%gu9n`R`#s|Iu6E5PZ=L4(42)kG&P+y(piKD{i(@;r4m zE%cjD*9}$o{LC+bWkH8^R@BiX@%Uh%VHVrz1|doOusf7*375?7A&n`PD5lTZL3gMB zz0pLo39jl&Q!(Ufs>bIn$PkA=@d*RxO zDs4yiZX-8u=7_q2&lGfEKY`I0msJCu8n@BNH|^|VMk>U=bk-Rsvv>y=Ys$UE zyueOYuQGCFp1G*Z5r`G8 z#xgl|c28Kknak5gg3Cu0DGFoHe|*1tPA>4-(0omN1k2=ve@TMGA389qSwR&S0cvQb zvWg|sjGWWToS;zZ`4+NO1fVckp#H9WD;j+{lUIW|_s9M9_k*#AV>``?(lFqE4Yqd~ z^azZT9J|~1pCoky`k%tk5=BXVh7vV|g~Ne5f_kEO5Ob43;eH8l8Gvg)4#sqU#%->;GVlOAr#sBCC zW-UsLrvQtlIcp{cRMRLC^frKL?w_7G=iopSBv^dh% zE74`n*2Ddo<5RyX%Z6n4FM3f=Ss-9YXBH|OfT;U%9ZQ{5Ye!Fb2K2FkA`%DpPw=sA5IRx(2+#N$ukE=(HF?gXw>sv9{2nfngB2avx zTjVfTD}BqQ$-!?`Iu_Ubw|yXL63~0! z#Q>ItB^?`V!;-YF6hpr84Pt9B4ncPykPE{|_>c!NU0UHBQ=~MNt!&A8h?HSBtZ3&# z-Nv6TfR^AJN(LZMJTM%Xn*KYjY~H0bpB08nv_V*Zx&|G!h(+Q{6>G&3tpTzeWTFd4F&V=OXP~{weXnsP982G2y^dT1sdO(n!mnE%uBqCmN})YERP?dc9;*mrPhKD zG%KAO(}%--tsGh!bJFDZl-5bhwVaCU2jb?4IB$?DVoyJ&0>xfJs=22Cys_z-!LXy; zFGPFnry9VJ3T4*^I-%9(sSB{9Bd+>KLa8%)Q~LEG4l0g zrV}1sdt%&jo}kMqKSko@b6+YYT&0=|pM7uvGxP8~m7LT(dz^Tw6Oyc*3RVTZaUx1W zF%#$5j+igR)v3IoGbXktWC~AT%l{gGxLYMmbSdC`DSVNaoBR4ulXy4ZTsZZz~9%&>fv zveik|=52-@NfvQ-8%CHsM?_-1KQ`C9hmTr>%YxsxySB)LR8Zyb zDQD>8KnI@)h1@WlK^S9}56z!>cl2AdoSI3$IxD^8diD6SAy3}vg9~H=tsoDN_V!hO zS@#kR#K(o*9;z&5tQ7YysSM5V+MYCk~WP@}|(7nDUiMX*}#7LVhX)aeHd(lRwQUX?FLnxo~)PiEO1kqPhi)qB^Xv zozHIH6vRv+y|7XJqPsw{^AmnoTtp}|yFRDjTU$*L%1_~=xq&l$cc@mKKpN%)HJ!t8 zHRT@Kg|o%Pq|m?a%gQ8u0w_#Z8l#QTfq&AK;=A6;@i;=;5QH06=vP>Iock-Ltu4?! zLC;SX$W2I-2Zx0+Y_~Z%!sE6k-Y5w^Mok>rb8)cmbTB%Uwm7FYpW)83VrF{;^FvRR z@8h5wOyOA&8T<>iA`)FE_GFvn>jbgM`LIP`@J^XeN&KPh``1|iEw~Vj#)_v}tcv}9 z1MADC<|U^9wAl5!T_+(?XQadC%lYa#41Bi~AzB)JKEkbJhP8jg>Y#5KYwOci1A;A=rec+a5=ZIH%Ns1WHw>%!kdTqYM#K=Bd| z{Xeb)19(w{Q5deQuov|#dj)Z!b+$?HZHG$N8m98E9*Eaj0vNvzFPjP#swAJq>qCcxW}wka%?*WjbFDW4&lcA(LIKH_UPtBjErMq5w z@9S7%VzwB>IO9A;Dtm&sxz0~oXE1{^rXP)!0KLgn*6xt~VzAAsH}Btzx~8$K-T^jr z!86l1jBejyJjf3#3$TUYGuh?He_0n*{ug)bd-7 zCkcdMN@Q`}oL}g)F@BIkJb>;OCOiyHx6v^%%Q_f$LVdpRu7crSev+Kl@c;UqCJ?gzgiqqU{o1~#-`k2VaQ1JGp4^>T_ksFC>Ruv+ zHcQ~uO|vHDpvKD?WD!1PDRA|)0<<}r^x{$WcJ!M>c$p9g9Y?Gogc8kTXZ}_#i ze{1Nb+VLcAo$|Oi_vBZrbuc`RD-^so2b_tn0>iB}B+>0(VdXrS)W49zzL&W==26J4 zD$l0h<$1*gcJmKQ!1kZ$Y9AgI{Zs~J{2za)E&V3Jbd$S`g9KH+o);~#iX71Fu&X#8 zPl{~@cXR8GJ5F&fgTd=en*{U~tcn`~shw70m)7Rm%E0ygV1W%5>IKWuuN`F{*i^aO z4}Ay!n$J5tY02F~^e23eK(2>NJ>+8_byx}PuWjx6-cRpKAaZ|TX>TnyBs%{XS2YXO z(-Uur(_^=oXee4l5!RJ3-W9EQgk-RfL0*#0(Iz~8C3Z$B=j?tn0;+Q2sSBGJBGdm; z=?5(0=Zo)}3^-5(P10X1NZlckQq`2tXUlT#B&uHw+#^82%COn*zgr9NaYYwp;yB8{ zXPuA^>Dn?tzE&^Gj@)eOm_64|tmI0wArriaw5L}`8>nnwx;_0rDTNxjqXHpsxin`i zJVVfJb8g|UT>t1im{kBxtVTlX4Zc*Eo3=(ucip^?qQ*m+gRvw!Q7r*Rh+;bSPLT7l zv$j!anp=iP9;f4)b%`FB^%{1KDN^xzlQWB`_<;*;K@1kJL~i_1*ub9P$gN?S#w z{EZ4;tdjS?CT^N*qqi&fVGqT{2kJ_%8?^w8D|-3O0jZ#*oasfWMvYWTl!Xs#Ts;pHD8U+rA(z{?<@ub#6A} zX{IU|6$E+|Gd6%jM=aIYqe+-6idsWO{As_6TXx7AxRjow68fO+dW=pex+0!Z&3R1h z*g8$}4yR+_YfYjF5=~w-opV5XDt?Tmq!UzhLgAimOre8Zc+Znk)0sK@%JG^Lye|Ry zAKr(}tp@YPGTZa`-ralgLRXg99*pV|2Q!Xq+Lp#6{&!E&hzCOOG8D+dCgA}-0hXq| zbzBoWZ^kBRY>&6KNjaA-jsgkaW7ZUX`o3kd$3?O+N~KqB3ZJ&q2VTxm`5uTejcy7N zFt5=(&t)2uHr^TiF8q_fWI@O2IQP_~lxO8xWWefk)R~#HXG1PjeNkXIYw|5MAmgFV zFg4T;Czod?l)59~L?d$I`a%(T)o0xT=W4HcqIid+gHHU%cI{)&IO-V3YalSljT(qI zy^^E>4CX16@0GvYy@ws!f-E;{RlT1751PctLtoCkKqbxd+!Qa8n1hQY>CUzl~j6)*j71S0G4@q^Sd8i9hQPX(}9>sh}k~+JU!WU zmx_#0x}s4P=J>27G@lzIL>WXLhDZvwED&mP?VyoNjInhNj)ir;a&KJ3&29-Qn1J*? zvEq24-kqVT1&b*6lzUCwAF;=;ex(L|+UHG7Eo{}t|Dbz=l5)ZhM{wbWB_`eXJlkw# zzKIAY*ovE)S8hfH?Jx(qCUN7natYQk(yyBZy2Bx11J-fSLDEsLO9IWXgw&f!k z6pwK3apIUDZ>Nw>72VuXg(CbMOY%8KCiu%y3U4U&M|nOzV#^P*^oq^$EdfSp zwcI9XXU9r+6CpsLwdC6JGVM_rrlsI;=8^2uDPY47Zj-CsP{b z4U}Q-)8hK&eKSJ$o~Euq)evey>`h~q0khr*CXh;G0R`Qe^gOrhic?9TL@ey+QCh}E zAYK8DF+_btI|!&Z5-UxBa3qPEUwj^8tym`M7>R~=+lxw!8Wpl4xm30PJDcaAga`Ib=>|YB5G9olWq9VgOTP1*@VLwcJXikaz3!9I;VUaJma`zv^~+!eOUte!{V+`v;j<@LQ~E?wNk|bue-P-$JyhF z?gW5R=~Qm#9LErIN#mD3mRqA;raVzyoZF9LwJ!oVdXZM8a`q6`y>K(kJ`;;8+e|{7 zK5sCNp^=DkKqh8JN`zo~J$;FPzdM78kAaKp{uOu7sd#Kj+S~P<_K_vT*zvROmp&3| zGjuAV>N4o&yimE#zZ^Nh?5KPWh^*Y(q)H>a#ya4U-DA8l?FVa@Bc>$9{|C_D{uz0~ zdB#VB27Kd~mQXoAU+|bnI_Tu3kiS>y>Hl61Whhne`2X%1hgF?MCbUCnBbOZX2q`5; zFOlK#geUyKZaX3I607gI6WdxWdx;e7e(={j5Fdf!=*`)ocrNIX0+1w{}*tH-E+DkH_|ISLZK8tyUdP{l?e5u&#d&HZ0wivGZ z)7Ymjyxyq9EOHdD)vfbHb{e z3*i9j{muSQ>kAtH?`a5cqr2B)klB3zpI1n;h_-zbrP=-;^Slmq*U3N7s6ff@mxXnO zyR7ZY?d}Yv{habTPwfQm@tVm8&Z&&PceTk?{35aMlPNN?OtTHp&?&a}w@t;jOG695 zbnC(7%_olD2FX=5_(fEYbcm3LhURNQ2E(Oa@(?o1(@G2NJ&lwwgg8wuRB}kqtA8V) zJ@c`lmUZf|4Yy)wp+#7gRG`-Iw<;x1sl|H*3l9$v|9k1(8bPz(mf$QHIT?uamTmuc z9VJxE4I&yf)S~%{Kh57}jJ1Yr*zdmo*B#gri`iGaag2gh1^r@jdIom-^v}yC#YQlX z`)6TF#M9BTccsNBdgp!yFY3D>3+I_rF>tXaNV;KNKqB9;mIe$v?d3KewLo?mH|(_9 zAfWC&k0g{c=AN1`g&#DL==RwtEBUqw9F(I9>L2nCtjl+eiw(qZ%3^Kv;4VyMX`r?+ z_ED9ZwjanX(AZGu_T4oydhiQzHK3($A>Q-qeI_Jx))GU|qIRzo*XYD=Y|(IWY#4vdFK9G4T3~7x;pHClNNY zUIpwurPqe^w^hj3742~XvvBKV*wv_K=Nnu?lZ4nF-04#zkx%VHhWK>uvgw5XDKQJC z`XayHGUWKo<&rJx^2IbI-1pm(GOte+daqxA^bcOn-> z9OtFs)n;|+S85iKE60p&dYF2sRTq((Tpeh}GrgEL?F4Ts>8&-l8?zpma?N>}9T@Z& zwpoun+}ztdU)~n&{p`Ts*>xrBSeZ@@`F_m{Pb%dJgR=WNfg4IQb-{&3&M z7#DHGge0d0Vv98rfWP@`OQg(y>U1HDzb-A`P%GBn>|8k^PnK}+_wH%%l}W4)IZ+@Y z42K^i7~$}JjWjpkrX06(6n$uB)u=^Mc#4;BjJkUYWG9>}pmbifQ!_-A;5RRoGA!q{ zm!vXr#?E_rk_>8t^N}};-nr4VTS3vshs=#lELSmCWZS3BOX5JGl|l<#lW-<6U00=hN=?;8aCKfvYiSc2&MVsScq;vn_g+)>clx~Z zfHl?4q)%+ZmHkCWU&fhr4t`@}7)Qy@+jmgz$1(*&Mk5B!BU&zn0X(ss0PO72hG9JP zd31b?XTDd=n3^);#aMtK!@2jrB6lL$tf)Feb6d)Ih3t0i!-UQK<@r`(3}=2o!aLm) z0>!`h?0h>!d6x_7^Smq=k0atz<`5QM*&gDOknzb`l(9;3YY)2!c81fuS!G12rGRnM z`;{wrEi?DLsEf00mGykY(rR~tfAesW71I$RKLccD#FrRAwiZ=FZACJz-LZncJ7C&b zu@03Eet-RAfRw~uY|Qei4pV%qCbUJ$b<3isYovEqA)hRc{BN{oFs+l`KNA#5PN`xM z*{xrF$QNzS{=EVh*w5c!%%YiyyjAM1dGin?j27z0a6!B-iOwZL%~Cou zqg5_+6M{$)oAS7*?@nPn(Ue@OlxFf)3$+M-6oMU9i1j4$ojh|7_LN644?w}H(yq_l_l8zF&_)N%KNiRSvcM*W z&Rd5x;7$92(ecgTIa{)C%tXp%bVqqK=X6xiaK=!f>9Pd0zec(ICIhkP;+eUn!{_-;Hl#8$XmTWJx6itmPrf-l;$qhve+HO>&MLLn zX)~OS8F^W)$ZQDIA9>pfjB%-^^YiVRh}?XH>GN_HBa$ozgrNGO>0uxN`ug%QX$A_U z%|Dd1-HO~qW;9qOm%<@&W?qCF&GCvK;V4f)yrDB2=o>6m00;B7TQtKwi%rsWy&cXk zg65knS0_!M@vPLBZ0mu5MjGGlwBT+`iW8p7NBnX_M32BQs_`uCpBixq)f>eiJ#1rk zQO&nOvLOTL0Y9uBrpATgj}exCGQ?*kn~$pO$AZlwW`NH~Y)Sc=8E7Z9z>km|Z#B|< zO!%UlMt4|^>_#;9`Hd1hEjJIhJ>QmF^@ixa4K@&l0VT^s>Q)3(RK?C(zxPZ@1K2zk zA~;S>fhfzZR>9!hTa0c zoD$U(Ig`|PZKyvL*u)z&+XCIy_oIf*HCB6D|eI7;c^rY{*dZ zfc4MDtn}da+O(K+)RGBcbT>2+p)JrECvghCE1oq0O$}9b-n=n|`f7B;XW;e5(q9DQ z!EjF!=ede;Bdbz9Tz%JP#Sw1fo+#^eFqg#k&qfNSb)0Qw+X}$0LsbxQOYQRCO?Oyu z5o`FRu=S^MxBLuIJ#?Et*}1^|q6SS=p(${FoTYC$V14>%LaCS&Yrj8GqW*kGwkGB^ z6%fmRs|&Yb%{4QjmsGhP3Etop&u`u!{_ITXHZP7Y%Ubvk11_;;AgEwjp$nA2*csmq zpq6y#?g^AiWKcNgNGxe?lF<_eszkT`S+S=zzmqOdbENjU^HbW1wG3JwohVCYTl03e zRy%Bk5nEB=0!P&p6)i3Dx01UQ)`UauBmPJWtd=IFa1e*!=OIg@8vtGJ-N_0t2$nGX z8n3@=v~X`u>SMgg@gIkBlmE8@C? z(0hQ~G>O1W-cXgD)}gRn`DE?Wj4Y3kjCaNu{?DWUGQ1dJS}RR~AD=c8hVLGvWUmO~ zLx>fT0_8|8%bf%~{zmP^S+pa|^pQ4_CN8a9;oDGa&NV0zigiZIHU`RC@k8t7rTu_f z`&xhUSvV_sRaOljRwH86{dvVPSsuw4=BOrf@z$Id!>|R(AXJfDTSzDFa&2lZapLs} ziIPi@5%%C>4)L6bbw$xjr_0A-H>Ekpcan>!(1AR&O!2JA37&}y(Ax4({?^B&o zKaB8DBbO&+d21JziPqje199K&C-O#ZQHG*9bnXH6Hkh{nYYEhLvFzh<8y1hDfl7n= z0~P7;-td-Zm-JAee8I)x2$PbJ%&v7}cYSqZnP$?Bg>H>V@VSeR2>b0%q|!HES<8aI zJ&b{8m-h3S5}nvSI|&30-vPM{|K`qglEOKe32~o^l1M*S3AZ}o{K09SB#V0kLk_Tw zi%0+P4Eik}v4ZpW+1wcpUa8(pZ3!1!a6SXkLcg{!bY7?+-N`Shll~yEe0}~Q<)IATPyjUZ2kT>wR_EFg?rD~%;kzNOLNoydK|Q^p zvEhF@sn=YB-u@S`>xowa{oPBKm{SY`#Uqo}+ z$Ed!sS5MyOB9kJM++cjZ*ZGs`d!tv&#Lm4CDM(1gErl+(1h~R0>gDdmK2G2N?r>1Z z@EV6F4~poF;z~@UQo<{blwzoylz#&x;*@yBp)Ofj^#kP_z-N+`y=T1*wtqv>4delA zjX^$dl+N(ZujP`;UJF!MWb=4K;rS?p3Twhbz3T;q)UP$dz5%@%_B0=jXu`Vo#&x?CjjVyE= zD|Ttu@nUHah!}Kx9V^Jk$&u74+RZ<*|NusYh|KF{LzOQS~;oyB|~H1WW%^+<`%oQw+Tc) z9i2Nv1EdXWJ+54{-6N%O|J;P!*&Z1{wYNQ-OpOW)eW0hFw@|Fq`zO>wYb_0F&bVL* z2|Cm0bdTNH8}^bu%KFXgM@jksXr?GGTA)P^8nzQ6)WB9nEmPPw{u(!^rWvIOPsWnC zc#)6Peoc(5ADPK3GpACklF`N(bc28W2g#ukk8P!ib+WorNm#sqGl~_l+S0bPXsDKY z-ZuFk?t1%*RC@FQR<4Ty4i+2wAaD}r;l-P4X^eBGnZ!?`SZJYsJ+FIo+Nzh_$K{SP zL;g4O9#`K%7xY@2YU(IiwpcD+VL}1<>N%6<4fIB) zqy{u)S@s+wc4N)(!aN-$&{s5XtK1?@g28?n{!NGr!_SUIflr>Z2M7i_qHnc8m@rLr z8PDVC`eS?Bji=Zo3DA>eWHVTe6be9VSj-z{hI%~*g(SZ1>2Z|*G37 z{_5?!AZ^nG33lw@UMZXH_Q7lsC4usZNCs5w>!5x~S>do!V=2WXb0QZzlFW7oiZU=&)44fdh7v# zAb#Kee?~znGl!cZq<+IBc&YuC`NoU$kz#BQ)) z?oSl?Zjomn`%ZSn*qx~+<2h&R=%%mE?AF%1Qr`@jDuQw7IIRw zj)M~#!B)haMV*C+vh}_l`b5G4W^!d&bVdv909p3vnS9TN5M(#+0qY+W;Dw9mrzG$D ztWwpWa-U?M)o@X!>xAgrMKw~)LwRMC1M-p^+%Tz7+{B`8XHUC%WvrXvnP+p$J&Xj_l6bn~%DTFGV(2zA+$Xt8F!NCo8$dP@z6Pqlik?_U z%Z3h>n=cH}@|d6|0kubm`kmpj5Y&KEB5Srh9lQM(n1i(GN75I=9Kd5yBH0)kjkXj< z))4lSz+$dd#027zYuQdrf`FzIrRYT{cA~7xDpc+^s&%j2?t4O#*E78}PF99PPu=B4Q|OSjXg$Vvv-z_ z8{?F%y@A$7{|Ue%`XEOdMvR~-l&0U0OtN)vs5z@xHPFAZ5UAh=86iC6a8TCyFCYm* zQ)u&#dnhYVGt^#poZjPBAJk-wlYop6SHh%??8+99*yQq(Iv;QTc z2+ytek7#!Comb@EHa!|k(_AJV{xw>HUA-}<>67mmf=|=OPbl6g+`8pTwO7I-EOpHMCaZZZpU$#_v&^}AFyj6O zVM+-_Lua9HS5hhBpJ+3v2}LBkF(n})Ud4}-MQjmEay;o&A!%zPB0EZe7|WTIrxhLc z=B_X8->szyHv#EWB)bu)Pa0876iLVo+bN;SyJInfViJABWgJN0(Xs)apNYo+&o+3n zh-lQ)kK83PEGpp9?qx~JT8M59c=^3Ge=bv7%SYegU{fI|2k@40nDGkRaKfA9wBt_g zf?m|23Kd}dtNYt^p3N=J?cHSCYB{zoD60_Cf2=)ac9lsgI+AQ;>-j&7o$L0X~ow{;O3^LgLT3s}yYp^HGom9ALj3g1i z4D=8`u4;}`KUfBV9e9S*$~gxXk3Lm1 z2uX(PRqTo#695D8_}?U{@5{t3$a!G-ZYFORN33H{VO%rU)d-B1J6HSmKzv^vJ9a$Q z6)9J@ry@IY*u2FbM_z08jm27lU16oZfY<=;`~};u_r!{9Xn0ibqC{)5;kR&zov8*| z5l!*-BL7Mi0VYIzf%)9_m1n*s&crCYwmvw9^DFF4tl%oyew?mdRiqU|wV)XL&LpMw zzaH5J3cT#OQlH;qU%>P0Ql7$!l(=?)7a0)(z`d2RzXLe#d+THk1~h+w{H}YcwCpz!3dHi3`QjLv z{J2sE^`dWC3ny|XLwr5YlxGj&^1%{7V&K!tV~YUHlH6`nO#^oX^e#a=bVKSoJzmYS zx@5#<;8IarK0nu2)fdyV+(fTyj{PILG zjrw1+)o+#-O~+6^sYn*`Zi*c8H#gT+J9tL~;p}AAh@oh(4SP&Fharqh1>c9LEJ29fln4kY? zN7pG_Hk?W}n@Jzy|EAD#!yA`6Z|vdjv+0TEi;5oN11^3jni7z!ufF-z+VL|{>R?P1 zX7;2h^#G7oAiC9)X_y!%=?c;Cs}4rM>90&NE+b!HCQi4GGf8q1F6b$8Re(r%2R(ab z=J*sDl0S@10`qAa_||kcQMe;KndWn^G^p5XA($&izKWED7{;Isv16)!iU1o}yIWsB z;zS|uy}fGR)ZX4?Fpj)-I2aOurD3bsue*a_v?!1)`IOoC35T|@%N60MqlnmXiGtfv zAmz^`Nnqtwn}P6f2v^r~IjtE{Ef6kz$KJA0OJ@6Y0eVT?7Di%?+h5^t1OaxG3vT`{ zq!Z*u{h4tS#TKHvX-iPb)Y-F=1y51BT!GL$w*s|*%PfGs+1}dr@FS)BhY8#x_g8@? zp)0dKkHnkYhx~zzp1bNs4jzZGYT(r^qFG=zBji;>Mg9a&dczLKz=pX2rW;wIx-^-f!h!!jGxZ< zW*Iu`2U$#$9J@6`EB2Ta`L%t0`fCc&n}0WytnQ7k)iV#fjrXL;D8Gwi<}$--Z&o=Y zg=J3N6&QN!RrScaRYq$L5A+%Is(IKfV=ZYYsM!2o zN*Cwc@6@FhTAro!@!LV(7atUmCfxW90NTEhqFzrSX(MdwgKftW-zPWF;SO# zJ`?l${Lxa9VaFz-^jOesoO6nHM}Pn&@p9lpt{+!-Yqq?Xl;Z&mE<^8z_}|NogPhMY zsTpk|Mf;RMG@jwv)%ZnVVM}C$Nf&!8#S$zmTcjiV8 z%IeUPN+%k8#SLP4#e_S`h&NE>n_ULqTo+F=#Ue33qlF(r1Tm0;4IhOIfE&iRK)@u$ zOAZ6Q^iCAtTJ~XWwh3lo>yaTU`7Dv1KY7aHBtp^f@$y)ZKd^w)|AB9`rCb}u%B7a7 z0OUTY)v44cE5E8FVX!@7Qm66&hc8FGA?Q*!!_?>YmtGir@UwVPo5$YnTbv6fAfD+N z-zqfl&raJtJ=XeBa5w z=?L&WGtNLNc=Q!;(;X=)zHPMR=$U-Q>yjxuU5Z19vE0JVjtUmI%3r?d={gSJj=zQK zM#F|lNG?_GP%|gZSA(&)Bv{_ig+2QBUN$AFHIyw_5nSgtV>A|f4+dOUAQ8ode}=! z+^TTlfCKC2FWg`pQ(M1BP1+H(QSN5d_kmF zv}^nWxJ7UA7)AOCg3@n|;L>Mu8(iyQ=&}bj&^bM}`mIP;qLL14(I)mu@@@<7djq#b zYzWx-tSd+8oPWKq=(PO!AUx+d=MikL z3o1fOgg$Prh4WNU&Yaw)_*r&Z=WrVSW|8BxPm*ba!yb!NFKe~KD368}ou0PZfn)(4 zCBblzc;v9B)zDK%giPbaTS5plh4udvS*`e%yS%t@Z;-3i9f(4s@Kpgp9}x^MxVdLv zFaB}nHKOrxfYFdbS{1KF5KXm{1?X^}w9+59TMvfFlUx6R6zeE>9Q%$ZufR}s!h~q{ zCb8~$ChJ%Z`wJ7*ZcIa`g?fG9a+=-+!c|pK(|$;HYo?*!3$I%P^Hsnz^LKhuT7o6D zeo?_zIf!iq4>f6wD2EE7eQ@Lx7E+hj_m~o`5|Ma6)di>CK+`dlEC;h>%U!!0L+>-8 z6)J4(uW%+>l&;xq13#2Jn4*ksdBH}kN<#-X^Gqz_U^4~uj`ed7hu@!?+*vTbk=4IJ zsnCI-MDhypp(pqPJAZ{cNGFQ$I2hxPD@Pw2{y9t8aDJD;NoB9x2`Z_+ZVCyv&1%dT zEQ51U&Xt;rpS1|4MZwXKIRRQ6@HPehj!bf%N1c{Zp>n!mX?n!gH8EI&8nMR((q*~@ z1D2+X)^rYVE2=H#qyIaP?2q=yV8~kg91JwDcY%^WvRxsVnV47z6w$9mY(iSnN%%@} zX+Js*gCHzbASk-@q!lMEw|v`$q72c7=~3Gme(iaq*p4~7Z_DLgyd56*Q(0=^gM z|7$IC+um{gLwzJeGHP8wNhQ>4?>p%=SSFm9@0=JhQ~Tg+L^-m&QbdfD;x|KtYNa9^ zWFQkcbk;E?^M(JbCNTrA0?7}S6$F4z!R)tMdXN1@9cu&WU>NQQY(HI{({%w!hTfm> zioU!NP%52STwj_)|9g3RlE;O|v|?Snki%%NsJ~2DgN)y4Oo{nVmJD(lJP?7(E8#R; zzJ48DDR{^q)B4_1ER4&fnh*k8e8_#a2NiWFa-o)ts_n3^O2m!yWMlsvnwacltW z_o5HP9Gv4l$LF;E7ce}D6VK|6q8%W$aXEVKdzu=O9E^l7E4uE$vG@$@%{bxWl8wg( zkgpD|nvSeeouiw^jyVA;Q3f=saonG_LkQEl$4%!Nn!rw^e>(pqt6v+)J%qkq1#+fN z4j9<;Heq>(D9VK=2GCsK4O^i^;p6*5PEAS>TEeDcA~$~-bKR@wG5ywF@7R2Ky{dk6 zgdY^cGz+~sc}I2rMriwt*I}#ePo0cC^4CTGDvOZiGP|k5dA^au zPx0tQ3G#ccTpd(g*jqZfuKcgLIuslPErm?bi4K)S=sJ5=4ek0bS~IyB!xJ?WzJUbj5L@ z0U{Yv;aFZ`HPu-LYe3`Au+oWz8ZB}nk;ID_cI%SU>fx;V8SWw9CC)LrhgPQP%Z*e28^|B7>nO|lRdvHOs znQ%yET$FV#y(!E$A1^=5c;b_`P5*Z#(m~Pb?Fh;d4;>*%EDlXdj2?T2ML|UL3vREw z9N@4&B~Rahy#Ql{m!pASz#Ap<@RmQgsa`spCgV&Af@{XBa(bf9qT7C>Cy?SF1Eq3|R-K#gN@B z6+mwA!{=E?GJw;DdKw?#7|1?uvu;-W*;X5dwDMhk+14x!6?BACGAltmC7(AvTeb3t z;ZhcKrhKS(iBxz|dP3PQD0YKy-d9k?&Ltjwa6rf6gZ=f8?KUFZO zSuN{C#RGY(+^nVVXz5N^2iJkk8}lCD5s>J-H?jhVQ#rSZ1u!xC0UX#G6jAbS)+3G! zKaKZ6|5IDN@6*)vY>!%nxItpNm=W6E-KzCN0HsTQQ*J=tLL(d=8Q=&EsP!iWVST&} z!h9R{`S|C`l>G(3Jrw0aoV~FLLTOMCI~llvzMj$0r>F=AaexSw&)BLMQPXjbmmivP ziOY&0b6rHJOG5R!hqYmhksB*s_^d!}`e+7>mO7r`3xjE?&^@0r{jkY$-&5UYIvbLu zGRhe{T`_@z_<(9QXHe2SWVx@alGXCOBl!{#pwQWz{KegAC2FcX3@K$hkm{*N)|G8=b)y@o*wo;#4((@%^LMl3G0{%pZrTQ{zUjpwG}|u8^+0f zl@g#f;xryALTxUjnUQR(K6}xzl04*`TT3=e3AqWP`<*0oMUp#^y0sPo#L2Nayu;YU zVjH36$MS{5*p3F52!O|=WsowW6M-~ejwZPcKQ$G0!E7hB<4J{WF-(xx{}(P;M-mGx z>s^%Bgd_-TT0SQx9Dm}s8DELoC;oHSL(L6Pb9Xr9E(#37m-|!3^6ExUm)?T&xH7x4#gF@X zxf3F*#i^P8r3PRR;hL^icrvTnz*;Royj?V0BhRlCOk{sXA>QOkk5ix}%IVC|8Zpk+ zOM8mLHifc?2uKPp>OXlL+vG&oKb@{EqtrF8S!&~Pu=p5dc}3W0arVHH)RHttpAWn( zvpm}>6se}fBZ%f01uN#j@f-!M8hf98f@|i35&3S zvIvz0pDsQ3#kKv;N~0H*&xO}D#sQk#?E221G$hH87ijW>q^3GGlQ+rY*81*wU9?N3 zxoW2t?v%6DgrbO|!FyG`{at<*+E5XZ@v zxWmZ;@k-X8EE3bG#HKD0RJp^H{pG8y`u!V@3~9+kJ^Icu+>vW?pkFUTWt%2x1Td`P zkNy6wYe4VHEx&EGR1Y{$t+8==Iv@CKk}pM$TOzYnsFZqAYxCV`+zx#_IU za~&m&HomuuOe&6ETySA_N0^Kqd^}O`E)ByJT`5`@eDNBZxQImwK^z*Aw^w@s2F&F; zAR2)gxzXK+TW`a&EIRTtux-fvuc-liqd}Dyr}w##*P}9t-Uc#!)NkX?`I-e)Kb8(oiJx^|ZI4Mj{psp<0QKml z5H#9F;056udsJS`1jzN<0#EM6;xIr%J9hS!{N)ZYMpH`8$2KJf7zD_{8J4mR4Co@` zG6S`a6xGp|$;A(KNZ`k}M2OY}w*Q1sWb=dP(6+v8kw`iT!UkWGNGI*q+r+a(o#z?q z0K#T%YP=G9TAXcH!Ce9|6izk_AyM+hK8gO#we37YX+rL$n+!P~AhCLOqSI(v`os1* zZInN;Qkloai^ND@51d_`t`>d0rl^RDB6_=iOR#(Y=@i^f@L)8uzhCjL# ztZTR*b{$5UP`5?45f|a0X9Vss-svWkdpB^nE-id&+p(C!t_L6)9YgJjd52R!z}g$* znNrf}Rl>0}&xOU#K}|hnVDakU0zd-lL=5g@A_}oP37uS zhgmXp%Z;#JADOtZ+#wb2W*J&F1wPFMb{iMUXd7f)56=!5KC)q-hAI%!8DP+5D@AB$ zg!Gje0ou+Bvr}gs>BOM!w{K2sV%7xjTazmPYB)L(udiw0k9MU3p2#V(-E7(vzk?Zo z$k(6X%@`!_nbAkuBd!tr-%yz{`}wxN(SSPx=k1+7`ibOIU|hfstG_8@Ho5{HiMkX9 zuGT@X@cFVo@_T(~I-k9EJVoGzx&lFo`s(#|y@Wv^^wr)L4b6#OsFGr_00000Vq20200Gvq4S Date: Thu, 1 Jan 2015 14:00:34 +0100 Subject: [PATCH 0281/1392] Slightly improved loose object decompression test --- gitdb/test/test_stream.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index eab9a1950..aa434ad3b 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -16,7 +16,9 @@ DecompressMemMapReader, FDCompressedSha1Writer, LooseObjectDB, - Sha1Writer + Sha1Writer, + MemoryDB, + IStream, ) from gitdb.util import hex_to_bin @@ -27,6 +29,7 @@ import tempfile import os +from io import BytesIO class TestStream(TestBase): """Test stream classes""" @@ -144,6 +147,7 @@ def test_compressed_writer(self): def test_decompress_reader_special_case(self): odb = LooseObjectDB(fixture_path('objects')) + mdb = MemoryDB() for sha in ('888401851f15db0eed60eb1bc29dec5ddcace911', '7bb839852ed5e3a069966281bb08d50012fb309b',): ostream = odb.stream(hex_to_bin(sha)) @@ -151,4 +155,8 @@ def test_decompress_reader_special_case(self): # if there is a bug, we will be missing one byte exactly ! data = ostream.read() assert len(data) == ostream.size + + # Putting it back in should yield nothing new - after all, we have + dump = mdb.store(IStream(ostream.type, ostream.size, BytesIO(data))) + assert dump.hexsha == sha # end for each loose object sha to test From 0d22c80e041dbb5d9d985926b39b7bd7a0573a7a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jan 2015 16:00:55 +0100 Subject: [PATCH 0282/1392] Added integrity test for loose objects to search large datasets for the issue described in https://github.com/gitpython-developers/GitPython/issues/220 See test notes for proper usage, it all depends on a useful dataset with high entropy --- gitdb/test/performance/test_pack.py | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index 97c450d6b..d54a74cf7 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -9,6 +9,11 @@ TestBigRepoR ) +from gitdb import ( + MemoryDB, + IStream, +) +from gitdb.typ import str_blob_type from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB from gitdb.utils.compat import xrange @@ -70,6 +75,32 @@ def test_pack_random_access(self): total_kib = total_size / 1000 print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed), file=sys.stderr) + @skip_on_travis_ci + def test_loose_correctness(self): + """based on the pack(s) of our packed object DB, we will just copy and verify all objects in the back + into the loose object db (memory). + This should help finding dormant issues like this one https://github.com/gitpython-developers/GitPython/issues/220 + faster + :note: It doesn't seem this test can find the issue unless the given pack contains highly compressed + data files, like archives.""" + pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + mdb = MemoryDB() + for c, sha in enumerate(pdb.sha_iter()): + ostream = pdb.stream(sha) + # the issue only showed on larger files which are hardly compressible ... + if ostream.type != str_blob_type: + continue + istream = IStream(ostream.type, ostream.size, ostream.stream) + mdb.store(istream) + assert istream.binsha == sha + # this can fail ... sometimes, so the packs dataset should be huge + assert len(mdb.stream(sha).read()) == ostream.size + + if c and c % 1000 == 0: + print("Verified %i loose object compression/decompression cycles" % c, file=sys.stderr) + mdb._cache.clear() + # end for each sha to copy + @skip_on_travis_ci def test_correctness(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) From 46a4a79f46ab5a8da97714262095318090674277 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jan 2015 16:12:15 +0100 Subject: [PATCH 0283/1392] Bumped new version Fixed tiny issue in python 3 --- doc/source/changes.rst | 8 ++++++++ gitdb/__init__.py | 2 +- gitdb/test/test_stream.py | 4 ++-- setup.py | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index f544f76c0..a36fd659c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,14 @@ Changelog ######### +***** +0.6.1 +***** + +* Fixed possibly critical error, see https://github.com/gitpython-developers/GitPython/issues/220 + + - However, it only seems to occour on high-entropy data and didn't reoccour after the fix + ***** 0.6.0 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 165993fc9..2a689400e 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -27,7 +27,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 0) +version_info = (0, 6, 1) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index aa434ad3b..44a557d53 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -148,8 +148,8 @@ def test_compressed_writer(self): def test_decompress_reader_special_case(self): odb = LooseObjectDB(fixture_path('objects')) mdb = MemoryDB() - for sha in ('888401851f15db0eed60eb1bc29dec5ddcace911', - '7bb839852ed5e3a069966281bb08d50012fb309b',): + for sha in (b'888401851f15db0eed60eb1bc29dec5ddcace911', + b'7bb839852ed5e3a069966281bb08d50012fb309b',): ostream = odb.stream(hex_to_bin(sha)) # if there is a bug, we will be missing one byte exactly ! diff --git a/setup.py b/setup.py index dc142c518..c4d9b2ac4 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 0) +version_info = (0, 6, 1) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext':build_ext_nofail}, From 8b4939630a0d7362e5a6fbca052922d710a87c7e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jan 2015 18:26:04 +0100 Subject: [PATCH 0284/1392] Improved decompression test to scan the entire git repository, instead of just packs This should make it easier to assert the issue is truly fixed now [skip ci] --- gitdb/test/performance/test_pack.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index d54a74cf7..e46031115 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -11,6 +11,7 @@ from gitdb import ( MemoryDB, + GitDB, IStream, ) from gitdb.typ import str_blob_type @@ -83,7 +84,8 @@ def test_loose_correctness(self): faster :note: It doesn't seem this test can find the issue unless the given pack contains highly compressed data files, like archives.""" - pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) + from gitdb.util import bin_to_hex + pdb = GitDB(os.path.join(self.gitrepopath, 'objects')) mdb = MemoryDB() for c, sha in enumerate(pdb.sha_iter()): ostream = pdb.stream(sha) @@ -92,7 +94,7 @@ def test_loose_correctness(self): continue istream = IStream(ostream.type, ostream.size, ostream.stream) mdb.store(istream) - assert istream.binsha == sha + assert istream.binsha == sha, "Failed on object %s" % bin_to_hex(sha).decode('ascii') # this can fail ... sometimes, so the packs dataset should be huge assert len(mdb.stream(sha).read()) == ostream.size From 84929ed811142e366d6c5916125302c1419acad6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 Jan 2015 11:19:57 +0100 Subject: [PATCH 0285/1392] Applied autopep8 autopep8 -v -j 8 --max-line-length 120 --in-place --recursive --- doc/source/conf.py | 7 ++- smmap/buf.py | 23 ++++---- smmap/exc.py | 2 + smmap/mman.py | 109 ++++++++++++++++++------------------ smmap/test/lib.py | 8 ++- smmap/test/test_buf.py | 20 +++---- smmap/test/test_mman.py | 59 +++++++++---------- smmap/test/test_tutorial.py | 8 +-- smmap/test/test_util.py | 20 +++---- smmap/util.py | 61 ++++++++++---------- 10 files changed, 165 insertions(+), 152 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 90409a138..5aa28e2a6 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -11,7 +11,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -172,8 +173,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'smmap.tex', u'smmap Documentation', - u'Sebastian Thiel', 'manual'), + ('index', 'smmap.tex', u'smmap Documentation', + u'Sebastian Thiel', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/smmap/buf.py b/smmap/buf.py index 66029cb99..17d2d369b 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -10,6 +10,7 @@ class SlidingWindowMapBuffer(object): + """A buffer like object which allows direct byte-wise object and slicing into memory of a mapped file. The mapping is controlled by the provided cursor. @@ -20,9 +21,9 @@ class SlidingWindowMapBuffer(object): underneath, it can unfortunately not be used in any non-pure python method which needs a buffer or string""" __slots__ = ( - '_c', # our cursor - '_size', # our supposed size - ) + '_c', # our cursor + '_size', # our supposed size + ) def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0): """Initalize the instance to operate on the given cursor. @@ -57,7 +58,7 @@ def __getitem__(self, i): if not c.includes_ofs(i): c.use_region(i, 1) # END handle region usage - return c.buffer()[i-c.ofs_begin()] + return c.buffer()[i - c.ofs_begin()] def __getslice__(self, i, j): c = self._c @@ -72,9 +73,9 @@ def __getslice__(self, i, j): j = self._size + j if (c.ofs_begin() <= i) and (j < c.ofs_end()): b = c.ofs_begin() - return c.buffer()[i-b:j-b] + return c.buffer()[i - b:j - b] else: - l = j-i # total length + l = j - i # total length ofs = i # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower # in the previous iteration of this code @@ -86,7 +87,7 @@ def __getslice__(self, i, j): ofs += len(d) l -= len(d) md.append(d) - #END while there are bytes to read + # END while there are bytes to read return bytes().join(md) # END fast or slow path #{ Interface @@ -100,7 +101,7 @@ def begin_access(self, cursor=None, offset=0, size=sys.maxsize, flags=0): :return: True if the buffer can be used""" if cursor: self._c = cursor - #END update our cursor + # END update our cursor # reuse existing cursors if possible if self._c is not None and self._c.is_associated(): @@ -112,9 +113,9 @@ def begin_access(self, cursor=None, offset=0, size=sys.maxsize, flags=0): # If not, the user is in trouble. if size > self._c.file_size(): size = self._c.file_size() - offset - #END handle size + # END handle size self._size = size - #END set size + # END set size return res # END use our cursor return False @@ -128,7 +129,7 @@ def end_access(self): self._size = 0 if self._c is not None: self._c.unuse_region() - #END unuse region + # END unuse region def cursor(self): """:return: the currently set cursor which provides access to the data""" diff --git a/smmap/exc.py b/smmap/exc.py index 5e90cf722..117664502 100644 --- a/smmap/exc.py +++ b/smmap/exc.py @@ -2,8 +2,10 @@ class MemoryManagerError(Exception): + """Base class for all exceptions thrown by the memory manager""" class RegionCollectionError(MemoryManagerError): + """Thrown if a memory region could not be collected, or if no region for collection was found""" diff --git a/smmap/mman.py b/smmap/mman.py index 6663687d0..c7a459558 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,12 +1,12 @@ """Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" from .util import ( - MapWindow, - MapRegion, - MapRegionList, - is_64_bit, - string_types, - buffer, - ) + MapWindow, + MapRegion, + MapRegionList, + is_64_bit, + string_types, + buffer, +) from weakref import ref import sys @@ -19,6 +19,7 @@ class WindowCursor(object): + """ Pointer into the mapped region of the memory manager, keeping the map alive until it is destroyed and no other client uses it. @@ -29,12 +30,12 @@ class WindowCursor(object): that it must be suited for the somewhat quite different sliding manager. It could be improved, but I see no real need to do so.""" __slots__ = ( - '_manager', # the manger keeping all file regions - '_rlist', # a regions list with regions for our file + '_manager', # the manger keeping all file regions + '_rlist', # a regions list with regions for our file '_region', # our current region or None '_ofs', # relative offset from the actually mapped area to our start area '_size' # maximum size we should provide - ) + ) def __init__(self, manager=None, regions=None): self._manager = manager @@ -65,8 +66,8 @@ def _destroy(self): # this python problem (for now). # The next step is to get rid of the error prone getrefcount alltogether. pass - #END exception handling - #END handle regions + # END exception handling + # END handle regions def _copy_from(self, rhs): """Copy all data from rhs into this instance, handles usage count""" @@ -121,11 +122,11 @@ def use_region(self, offset=0, size=0, flags=0): # offset too large ? if offset >= fsize: return self - #END handle offset + # END handle offset if need_region: self._region = man._obtain_region(self._rlist, offset, size, flags, False) - #END need region handling + # END need region handling self._region.increment_usage_count() self._ofs = offset - self._region._b @@ -221,13 +222,14 @@ def fd(self): :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), string_types()): raise ValueError("File descriptor queried although mapping was generated from path") - #END handle type + # END handle type return self._rlist.path_or_fd() #} END interface class StaticWindowMapManager(object): + """Provides a manager which will produce single size cursors that are allowed to always map the whole file. @@ -240,13 +242,13 @@ class StaticWindowMapManager(object): accommodate this fact""" __slots__ = [ - '_fdict', # mapping of path -> StorageHelper (of some kind - '_window_size', # maximum size of a window - '_max_memory_size', # maximum amount of memory we may allocate - '_max_handle_count', # maximum amount of handles to keep open - '_memory_size', # currently allocated memory size - '_handle_count', # amount of currently allocated file handles - ] + '_fdict', # mapping of path -> StorageHelper (of some kind + '_window_size', # maximum size of a window + '_max_memory_size', # maximum amount of memory we may allocate + '_max_handle_count', # maximum amount of handles to keep open + '_memory_size', # currently allocated memory size + '_handle_count', # amount of currently allocated file handles + ] #{ Configuration MapRegionListCls = MapRegionList @@ -279,7 +281,7 @@ def __init__(self, window_size=0, max_memory_size=0, max_open_handles=sys.maxsiz coeff = 64 if is_64_bit(): coeff = 1024 - #END handle arch + # END handle arch self._window_size = coeff * self._MB_in_bytes # END handle max window size @@ -287,9 +289,9 @@ def __init__(self, window_size=0, max_memory_size=0, max_open_handles=sys.maxsiz coeff = 1024 if is_64_bit(): coeff = 8192 - #END handle arch + # END handle arch self._max_memory_size = coeff * self._MB_in_bytes - #END handle max memory size + # END handle max memory size #{ Internal Methods @@ -310,23 +312,23 @@ def _collect_lru_region(self, size): for regions in self._fdict.values(): for region in regions: # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-2 == 0 and - (lru_region is None or region._uc < lru_region._uc)): + if (region.client_count() - 2 == 0 and + (lru_region is None or region._uc < lru_region._uc)): lru_region = region lru_list = regions # END update lru_region - #END for each region - #END for each regions list + # END for each region + # END for each regions list if lru_region is None: break - #END handle region not found + # END handle region not found num_found += 1 del(lru_list[lru_list.index(lru_region)]) self._memory_size -= lru_region.size() self._handle_count -= 1 - #END while there is more memory to free + # END while there is more memory to free return num_found def _obtain_region(self, a, offset, size, flags, is_recursive): @@ -336,7 +338,7 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): :return: The newly created region""" if self._memory_size + size > self._max_memory_size: self._collect_lru_region(size) - #END handle collection + # END handle collection r = None if a: @@ -354,10 +356,10 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it raise - #END handle existing recursion + # END handle existing recursion self._collect_lru_region(0) return self._obtain_region(a, offset, size, flags, True) - #END handle exceptions + # END handle exceptions self._handle_count += 1 self._memory_size += r.size() @@ -404,7 +406,7 @@ def num_file_handles(self): def num_open_files(self): """Amount of opened files in the system""" - return reduce(lambda x, y: x+y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) + return reduce(lambda x, y: x + y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) def window_size(self): """:return: size of each window when allocating new regions""" @@ -441,7 +443,7 @@ def force_map_handle_removal_win(self, base_path): **Note:** does nothing on non-windows platforms""" if sys.platform != 'win32': return - #END early bailout + # END early bailout num_closed = 0 for path, rlist in self._fdict.items(): @@ -449,13 +451,14 @@ def force_map_handle_removal_win(self, base_path): for region in rlist: region._mf.close() num_closed += 1 - #END path matches - #END for each path + # END path matches + # END for each path return num_closed #} END special purpose interface class SlidingWindowMapManager(StaticWindowMapManager): + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. Once a certain memory limit is reached globally, or if there cannot be more open file handles @@ -482,18 +485,18 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): lo = 0 hi = len(a) while lo < hi: - mid = (lo+hi)//2 + mid = (lo + hi) // 2 ofs = a[mid]._b if ofs <= offset: if a[mid].includes_ofs(offset): r = a[mid] break - #END have region - lo = mid+1 + # END have region + lo = mid + 1 else: hi = mid - #END handle position - #END while bisecting + # END handle position + # END while bisecting if r is None: window_size = self._window_size @@ -506,7 +509,7 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # Save calls ! if self._memory_size + window_size > self._max_memory_size: self._collect_lru_region(window_size) - #END handle collection + # END handle collection # we assume the list remains sorted by offset insert_pos = 0 @@ -514,7 +517,7 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): if len_regions == 1: if a[0]._b <= offset: insert_pos = 1 - #END maintain sort + # END maintain sort else: # find insert position insert_pos = len_regions @@ -522,8 +525,8 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): if region._b > offset: insert_pos = i break - #END if insert position is correct - #END for each region + # END if insert position is correct + # END for each region # END obtain insert pos # adjust the actual offset and size values to create the largest @@ -531,13 +534,13 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): if insert_pos == 0: if len_regions: right = self.MapWindowCls.from_region(a[insert_pos]) - #END adjust right side + # END adjust right side else: if insert_pos != len_regions: right = self.MapWindowCls.from_region(a[insert_pos]) # END adjust right window left = self.MapWindowCls.from_region(a[insert_pos - 1]) - #END adjust surrounding windows + # END adjust surrounding windows mid.extend_left_to(left, window_size) mid.extend_right_to(right, window_size) @@ -546,13 +549,13 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # it can happen that we align beyond the end of the file if mid.ofs_end() > right.ofs: mid.size = right.ofs - mid.ofs - #END readjust size + # END readjust size # insert new region at the right offset to keep the order try: if self._handle_count >= self._max_handle_count: raise Exception - #END assert own imposed max file handles + # END assert own imposed max file handles r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit @@ -563,10 +566,10 @@ def _obtain_region(self, a, offset, size, flags, is_recursive): # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it raise - #END handle existing recursion + # END handle existing recursion self._collect_lru_region(0) return self._obtain_region(a, offset, size, flags, True) - #END handle exceptions + # END handle exceptions self._handle_count += 1 self._memory_size += r.size() diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 67aec6333..93cb09a5e 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -9,6 +9,7 @@ #{ Utilities class FileCreator(object): + """A instance which creates a temporary file with a prefix and a given size and provides this info to the user. Once it gets deleted, it will remove the temporary file as well.""" @@ -21,7 +22,7 @@ def __init__(self, size, prefix=''): self._size = size fp = open(self._path, "wb") - fp.seek(size-1) + fp.seek(size - 1) fp.write(b'1') fp.close() @@ -32,7 +33,7 @@ def __del__(self): os.remove(self.path) except OSError: pass - #END exception handling + # END exception handling @property def path(self): @@ -46,6 +47,7 @@ def size(self): class TestBase(TestCase): + """Foundation used by all tests""" #{ Configuration @@ -58,7 +60,7 @@ def setUpAll(cls): # nothing for now pass - #END overrides + # END overrides #{ Interface diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d07b7f4eb..03377154b 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -3,9 +3,9 @@ from .lib import TestBase, FileCreator from smmap.mman import ( - SlidingWindowMapManager, - StaticWindowMapManager - ) + SlidingWindowMapManager, + StaticWindowMapManager +) from smmap.buf import SlidingWindowMapBuffer from random import randint @@ -57,11 +57,11 @@ def test_basics(self): with open(fc.path, 'rb') as fp: data = fp.read() assert data[offset] == buf[0] - assert data[offset:offset*2] == buf[0:offset] + assert data[offset:offset * 2] == buf[0:offset] # negative indices, partial slices - assert buf[-1] == buf[len(buf)-1] - assert buf[-10:] == buf[len(buf)-10:len(buf)] + assert buf[-1] == buf[len(buf) - 1] + assert buf[-10:] == buf[len(buf) - 10:len(buf)] # end access makes its cursor invalid buf.end_access() @@ -97,7 +97,7 @@ def test_basics(self): buf.begin_access() while num_accesses_left: num_accesses_left -= 1 - if access_mode: # multi + if access_mode: # multi ofs_start = randint(0, fsize) ofs_end = randint(ofs_start, fsize) d = buf[ofs_start:ofs_end] @@ -108,7 +108,7 @@ def test_basics(self): pos = randint(0, fsize) assert buf[pos] == data[pos] num_bytes += 1 - #END handle mode + # END handle mode # END handle num accesses buf.end_access() @@ -116,10 +116,10 @@ def test_basics(self): assert manager.collect() assert manager.num_file_handles() == 0 elapsed = max(time() - st, 0.001) # prevent zero division errors on windows - mb = float(1000*1000) + mb = float(1000 * 1000) mode_str = (access_mode and "slice") or "single byte" print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" - % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed), + % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), file=sys.stderr) # END handle access mode # END for each manager diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index d903af681..b718b065f 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -3,10 +3,10 @@ from .lib import TestBase, FileCreator from smmap.mman import ( - WindowCursor, - SlidingWindowMapManager, - StaticWindowMapManager - ) + WindowCursor, + SlidingWindowMapManager, + StaticWindowMapManager +) from smmap.util import align_to_mmap from random import randint @@ -29,7 +29,7 @@ def test_cursor(self): cv = man.make_cursor(fc.path) assert not cv.is_valid() # no region mapped yet - assert cv.is_associated()# but it know where to map it from + assert cv.is_associated() # but it know where to map it from assert cv.file_size() == fc.size assert cv.path() == fc.path @@ -60,7 +60,7 @@ def test_memory_manager(self): winsize_cmp_val = 0 if isinstance(man, StaticWindowMapManager): winsize_cmp_val = -1 - #END handle window size + # END handle window size assert man.window_size() > winsize_cmp_val assert man.mapped_memory_size() == 0 assert man.max_mapped_memory_size() > 0 @@ -89,8 +89,8 @@ def test_memory_manager(self): self.assertRaises(ValueError, c.path) else: self.assertRaises(ValueError, c.fd) - #END handle value error - #END for each input + # END handle value error + # END for each input os.close(fd) # END for each manager type @@ -101,7 +101,7 @@ def test_memman_operation(self): data = fp.read() fd = os.open(fc.path, os.O_RDONLY) max_num_handles = 15 - #small_size = + # small_size = for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): for item in (fc.path, fd): @@ -120,22 +120,23 @@ def test_memman_operation(self): size = man.window_size() // 2 assert c.use_region(base_offset, size).is_valid() rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us + assert rr().client_count() == 2 # the manager and the cursor and us assert man.num_open_files() == 1 assert man.num_file_handles() == 1 assert man.mapped_memory_size() == rr().size() - #assert c.size() == size # the cursor may overallocate in its static version + # assert c.size() == size # the cursor may overallocate in its static version assert c.ofs_begin() == base_offset assert rr().ofs_begin() == 0 # it was aligned and expanded if man.window_size(): - assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) + # but isn't larger than the max window (aligned) + assert rr().size() == align_to_mmap(man.window_size(), True) else: assert rr().size() == fc.size - #END ignore static managers which dont use windows and are aligned to file boundaries + # END ignore static managers which dont use windows and are aligned to file boundaries - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] # obtain second window, which spans the first part of the file - it is a still the same window nsize = (size or fc.size) - 10 @@ -153,16 +154,16 @@ def test_memman_operation(self): if man.window_size(): assert man.num_file_handles() == 2 assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager else: assert c.size() < fc.size - #END ignore static managers which only have one handle per file + # END ignore static managers which only have one handle per file rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] # unising a region makes the cursor invalid c.unuse_region() @@ -171,7 +172,7 @@ def test_memman_operation(self): # but doesn't change anything regarding the handle count - we cache it and only # remove mapped regions if we have to assert man.num_file_handles() == 2 - #END ignore this for static managers + # END ignore this for static managers # iterate through the windows, verify data contents # this will trigger map collection after a while @@ -193,21 +194,21 @@ def test_memman_operation(self): # precondition if man.window_size(): assert max_mapped_memory_size >= mapped_memory_size() - #END statics will overshoot, which is fine + # END statics will overshoot, which is fine assert max_file_handles >= num_file_handles() assert c.use_region(base_offset, (size or c.size())).is_valid() csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset+csize] + assert c.buffer()[:] == data[base_offset:base_offset + csize] memory_read += csize assert includes_ofs(base_offset) - assert includes_ofs(base_offset+csize-1) - assert not includes_ofs(base_offset+csize) + assert includes_ofs(base_offset + csize - 1) + assert not includes_ofs(base_offset + csize) # END while we should do an access - elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows mb = float(1000 * 1000) print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed), + % (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed), file=sys.stderr) # an offset as large as the size doesn't work ! @@ -217,6 +218,6 @@ def test_memman_operation(self): assert man.num_file_handles() assert man.collect() assert man.num_file_handles() == 0 - #END for each item + # END for each item # END for each manager type os.close(fd) diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index 5c931de73..f7a2128a9 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -20,7 +20,7 @@ def test_example(self): # Cursors ########## import smmap.test.lib - fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + fc = smmap.test.lib.FileCreator(1024 * 1024 * 8, "test_file") # obtain a cursor to access some file. c = mman.make_cursor(fc.path) @@ -40,7 +40,7 @@ def test_example(self): assert c.size() c.buffer()[0] # first byte c.buffer()[1:10] # first 9 bytes - c.buffer()[c.size()-1] # last byte + c.buffer()[c.size() - 1] # last byte # its recommended not to create big slices when feeding the buffer # into consumers (e.g. struct or zlib). @@ -72,8 +72,8 @@ def test_example(self): assert buf.cursor().is_valid() buf[0] # access the first byte - buf[-1] # access the last ten bytes on the file - buf[-10:]# access the last ten bytes + buf[-1] # access the last ten bytes on the file + buf[-10:] # access the last ten bytes # If you want to keep the instance between different accesses, use the # dedicated methods diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 745fedf2e..0bbf91b65 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,13 +1,13 @@ from .lib import TestBase, FileCreator from smmap.util import ( - MapWindow, - MapRegion, - MapRegionList, - ALLOCATIONGRANULARITY, - is_64_bit, - align_to_mmap - ) + MapWindow, + MapRegion, + MapRegionList, + ALLOCATIONGRANULARITY, + is_64_bit, + align_to_mmap +) import os import sys @@ -74,7 +74,7 @@ def test_region(self): assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size - assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size) assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) # with the values we have, this test only works on windows where an alignment # size of 4096 is assumed. @@ -83,7 +83,7 @@ def test_region(self): # argument of mmap. if sys.platform != 'win32': assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) - #END handle platforms + # END handle platforms # auto-refcount assert rfull.client_count() == 1 @@ -111,7 +111,7 @@ def test_region_list(self): assert len(ml) == 0 assert ml.path_or_fd() == item assert ml.file_size() == fc.size - #END handle input + # END handle input os.close(fd) def test_util(self): diff --git a/smmap/util.py b/smmap/util.py index 394e6b197..e079a527a 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -10,10 +10,10 @@ # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly # useful for aligning the offset. The offset argument doesn't exist there though from mmap import PAGESIZE as ALLOCATIONGRANULARITY -#END handle pythons missing quality assurance +# END handle pythons missing quality assurance __all__ = ["align_to_mmap", "is_64_bit", "buffer", - "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] + "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities @@ -25,7 +25,7 @@ def buffer(obj, offset, size): # return memoryview(obj)[offset:offset+size] # doing it directly is much faster ! - return obj[offset:offset+size] + return obj[offset:offset + size] def string_types(): @@ -45,7 +45,7 @@ def align_to_mmap(num, round_up): res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY if round_up and (res != num): res += ALLOCATIONGRANULARITY - #END handle size + # END handle size return res @@ -59,11 +59,12 @@ def is_64_bit(): #{ Utility Classes class MapWindow(object): + """Utility type which is used to snap windows towards each other, and to adjust their size""" __slots__ = ( - 'ofs', # offset into the file in bytes - 'size' # size of the window in bytes - ) + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) def __init__(self, offset, size): self.ofs = offset @@ -104,21 +105,22 @@ def extend_right_to(self, window, max_size): class MapRegion(object): + """Defines a mapped region of memory, aligned to pagesizes **Note:** deallocates used region automatically on destruction""" __slots__ = [ - '_b', # beginning of mapping - '_mf', # mapped memory chunk (as returned by mmap) - '_uc', # total amount of usages - '_size', # cached size of our memory map - '__weakref__' - ] + '_b', # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_uc', # total amount of usages + '_size', # cached size of our memory map + '__weakref__' + ] _need_compat_layer = sys.version_info[0] < 3 and sys.version_info[1] < 6 if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset - #END handle additional slot + # END handle additional slot #{ Configuration # Used for testing only. If True, all data will be loaded into memory at once. @@ -142,7 +144,7 @@ def __init__(self, path_or_fd, ofs, size, flags=0): fd = path_or_fd else: fd = os.open(path_or_fd, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags) - #END handle fd + # END handle fd try: kwargs = dict(access=ACCESS_READ, offset=ofs) @@ -162,18 +164,18 @@ def __init__(self, path_or_fd, ofs, size, flags=0): self._mf = self._read_into_memory(fd, ofs, actual_size) else: self._mf = mmap(fd, actual_size, **kwargs) - #END handle memory mode + # END handle memory mode self._size = len(self._mf) if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, self._size) - #END handle buffer wrapping + # END handle buffer wrapping finally: if isinstance(path_or_fd, string_types()): os.close(fd) - #END only close it if we opened it - #END close file handle + # END only close it if we opened it + # END close file handle def _read_into_memory(self, fd, offset, size): """:return: string data as read from the given file descriptor, offset and size """ @@ -181,11 +183,11 @@ def _read_into_memory(self, fd, offset, size): mf = '' bytes_todo = size while bytes_todo: - chunk = 1024*1024 + chunk = 1024 * 1024 d = os.read(fd, chunk) bytes_todo -= len(d) mf += d - #END loop copy items + # END loop copy items return mf def __repr__(self): @@ -221,7 +223,7 @@ def client_count(self): """:return: number of clients currently using this region""" from sys import getrefcount # -1: self on stack, -1 self in this method, -1 self in getrefcount - return getrefcount(self)-3 + return getrefcount(self) - 3 def usage_count(self): """:return: amount of usages so far""" @@ -245,17 +247,18 @@ def buffer(self): def includes_ofs(self, ofs): return self._b <= ofs < self._size - #END handle compat layer + # END handle compat layer #} END interface class MapRegionList(list): + """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( - '_path_or_fd', # path or file descriptor which is mapped by all our regions - '_file_size' # total size of the file we map - ) + '_path_or_fd', # path or file descriptor which is mapped by all our regions + '_file_size' # total size of the file we map + ) def __new__(cls, path): return super(MapRegionList, cls).__new__(cls) @@ -267,7 +270,7 @@ def __init__(self, path_or_fd): def client_count(self): """:return: amount of clients which hold a reference to this instance""" from sys import getrefcount - return getrefcount(self)-3 + return getrefcount(self) - 3 def path_or_fd(self): """:return: path or file descriptor we are attached to""" @@ -280,8 +283,8 @@ def file_size(self): self._file_size = os.stat(self._path_or_fd).st_size else: self._file_size = os.fstat(self._path_or_fd).st_size - #END handle path type - #END update file size + # END handle path type + # END update file size return self._file_size #} END utility classes From ff7615321ee31d981a171f7677a56a971c554059 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 Jan 2015 11:21:36 +0100 Subject: [PATCH 0286/1392] Applied autopep8 autopep8 -v -j 8 --max-line-length 120 --in-place --recursive --- doc/source/conf.py | 7 +- gitdb/__init__.py | 6 +- gitdb/base.py | 20 +- gitdb/db/base.py | 12 +- gitdb/db/git.py | 5 +- gitdb/db/loose.py | 8 +- gitdb/db/mem.py | 3 +- gitdb/db/pack.py | 6 +- gitdb/db/ref.py | 2 + gitdb/exc.py | 14 ++ gitdb/ext/smmap | 2 +- gitdb/fun.py | 104 ++++++----- gitdb/pack.py | 111 ++++++----- gitdb/stream.py | 52 +++--- gitdb/test/db/lib.py | 4 +- gitdb/test/db/test_git.py | 8 +- gitdb/test/db/test_loose.py | 3 +- gitdb/test/db/test_mem.py | 1 + gitdb/test/db/test_pack.py | 6 +- gitdb/test/db/test_ref.py | 5 +- gitdb/test/lib.py | 32 ++-- gitdb/test/performance/__init__.py | 1 - gitdb/test/performance/lib.py | 20 +- gitdb/test/performance/test_pack.py | 34 ++-- gitdb/test/performance/test_pack_streaming.py | 33 ++-- gitdb/test/performance/test_stream.py | 49 ++--- gitdb/test/test_base.py | 15 +- gitdb/test/test_example.py | 3 +- gitdb/test/test_pack.py | 34 ++-- gitdb/test/test_stream.py | 12 +- gitdb/test/test_util.py | 2 +- gitdb/typ.py | 6 +- gitdb/util.py | 34 ++-- gitdb/utils/compat.py | 2 +- gitdb/utils/encoding.py | 2 + setup.py | 174 +++++++++--------- 36 files changed, 460 insertions(+), 372 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 723a34503..68d9a3fcd 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -11,7 +11,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -171,8 +172,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'GitDB.tex', u'GitDB Documentation', - u'Sebastian Thiel', 'manual'), + ('index', 'GitDB.tex', u'GitDB Documentation', + u'Sebastian Thiel', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 2a689400e..791a2ef2f 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -8,6 +8,8 @@ import os #{ Initialization + + def _init_externals(): """Initialize external projects by putting them into the path""" for module in ('smmap',): @@ -17,8 +19,8 @@ def _init_externals(): __import__(module) except ImportError: raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) - #END verify import - #END handel imports + # END verify import + # END handel imports #} END initialization diff --git a/gitdb/base.py b/gitdb/base.py index a33fb67b9..5760b8af0 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -11,12 +11,14 @@ ) __all__ = ('OInfo', 'OPackInfo', 'ODeltaPackInfo', - 'OStream', 'OPackStream', 'ODeltaPackStream', - 'IStream', 'InvalidOInfo', 'InvalidOStream' ) + 'OStream', 'OPackStream', 'ODeltaPackStream', + 'IStream', 'InvalidOInfo', 'InvalidOStream') #{ ODB Bases + class OInfo(tuple): + """Carries information about an object in an ODB, provding information about the binary sha of the object, the type_string as well as the uncompressed size in bytes. @@ -62,6 +64,7 @@ def size(self): class OPackInfo(tuple): + """As OInfo, but provides a type_id property to retrieve the numerical type id, and does not include a sha. @@ -71,7 +74,7 @@ class OPackInfo(tuple): __slots__ = tuple() def __new__(cls, packoffset, type, size): - return tuple.__new__(cls, (packoffset,type, size)) + return tuple.__new__(cls, (packoffset, type, size)) def __init__(self, *args): tuple.__init__(self) @@ -98,6 +101,7 @@ def size(self): class ODeltaPackInfo(OPackInfo): + """Adds delta specific information, Either the 20 byte sha which points to some object in the database, or the negative offset from the pack_offset, so that pack_offset - delta_info yields @@ -115,6 +119,7 @@ def delta_info(self): class OStream(OInfo): + """Base for object streams retrieved from the database, providing additional information about the stream. Generally, ODB streams are read-only as objects are immutable""" @@ -124,7 +129,6 @@ def __new__(cls, sha, type, size, stream, *args, **kwargs): """Helps with the initialization of subclasses""" return tuple.__new__(cls, (sha, type, size, stream)) - def __init__(self, *args, **kwargs): tuple.__init__(self) @@ -141,6 +145,7 @@ def stream(self): class ODeltaStream(OStream): + """Uses size info of its stream, delaying reads""" def __new__(cls, sha, type, size, stream, *args, **kwargs): @@ -157,6 +162,7 @@ def size(self): class OPackStream(OPackInfo): + """Next to pack object information, a stream outputting an undeltified base object is provided""" __slots__ = tuple() @@ -176,13 +182,13 @@ def stream(self): class ODeltaPackStream(ODeltaPackInfo): + """Provides a stream outputting the uncompressed offset delta information""" __slots__ = tuple() def __new__(cls, packoffset, type, size, delta_info, stream): return tuple.__new__(cls, (packoffset, type, size, delta_info, stream)) - #{ Stream Reader Interface def read(self, size=-1): return self[4].read(size) @@ -194,6 +200,7 @@ def stream(self): class IStream(list): + """Represents an input content stream to be fed into the ODB. It is mutable to allow the ODB to record information about the operations outcome right in this instance. @@ -246,7 +253,6 @@ def _binsha(self): binsha = property(_binsha, _set_binsha) - def _type(self): return self[1] @@ -275,6 +281,7 @@ def _set_stream(self, stream): class InvalidOInfo(tuple): + """Carries information about a sha identifying an object which is invalid in the queried database. The exception attribute provides more information about the cause of the issue""" @@ -301,6 +308,7 @@ def error(self): class InvalidOStream(InvalidOInfo): + """Carries information about an invalid ODB stream""" __slots__ = tuple() diff --git a/gitdb/db/base.py b/gitdb/db/base.py index a670eea63..2615b13cc 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -19,11 +19,11 @@ from functools import reduce - __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') class ObjectDBR(object): + """Defines an interface for object database lookup. Objects are identified either by their 20 byte bin sha""" @@ -61,6 +61,7 @@ def sha_iter(self): class ObjectDBW(object): + """Defines an interface to create objects in the database""" def __init__(self, *args, **kwargs): @@ -100,6 +101,7 @@ def store(self, istream): class FileDBBase(object): + """Provides basic facilities to retrieve files of interest, including caching facilities to help mapping hexsha's to objects""" @@ -113,7 +115,6 @@ def __init__(self, root_path): super(FileDBBase, self).__init__() self._root_path = root_path - #{ Interface def root_path(self): """:return: path at which this db operates""" @@ -128,6 +129,7 @@ def db_path(self, rela_path): class CachingDB(object): + """A database which uses caches to speed-up access""" #{ Interface @@ -143,8 +145,6 @@ def update_cache(self, force=False): # END interface - - def _databases_recursive(database, output): """Fill output list with database from db, in order. Deals with Loose, Packed and compound databases.""" @@ -159,10 +159,12 @@ def _databases_recursive(database, output): class CompoundDB(ObjectDBR, LazyMixin, CachingDB): + """A database which delegates calls to sub-databases. Databases are stored in the lazy-loaded _dbs attribute. Define _set_cache_ to update it with your databases""" + def _set_cache_(self, attr): if attr == '_dbs': self._dbs = list() @@ -207,7 +209,7 @@ def stream(self, sha): def size(self): """:return: total size of all contained databases""" - return reduce(lambda x,y: x+y, (db.size() for db in self._dbs), 0) + return reduce(lambda x, y: x + y, (db.size() for db in self._dbs), 0) def sha_iter(self): return chain(*(db.sha_iter() for db in self._dbs)) diff --git a/gitdb/db/git.py b/gitdb/db/git.py index d22e3f1b9..a4f6f54c3 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -20,6 +20,7 @@ class GitDB(FileDBBase, ObjectDBW, CompoundDB): + """A git-style object database, which contains all objects in the 'objects' subdirectory""" # Configuration @@ -41,8 +42,8 @@ def _set_cache_(self, attr): self._dbs = list() loose_db = None for subpath, dbcls in ((self.packs_dir, self.PackDBCls), - (self.loose_dir, self.LooseDBCls), - (self.alternates_dir, self.ReferenceDBCls)): + (self.loose_dir, self.LooseDBCls), + (self.alternates_dir, self.ReferenceDBCls)): path = self.db_path(subpath) if os.path.exists(path): self._dbs.append(dbcls(path)) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 374302611..e924080ef 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -57,10 +57,11 @@ import os -__all__ = ( 'LooseObjectDB', ) +__all__ = ('LooseObjectDB', ) class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): + """A database which operates on loose object files""" # CONFIGURATION @@ -73,7 +74,6 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): if os.name == 'nt': new_objects_mode = int("644", 8) - def __init__(self, root_path): super(LooseObjectDB, self).__init__(root_path) self._hexsha_to_file = dict() @@ -164,7 +164,7 @@ def info(self, sha): def stream(self, sha): m = self._map_loose_object(sha) - type, size, stream = DecompressMemMapReader.new(m, close_on_deletion = True) + type, size, stream = DecompressMemMapReader.new(m, close_on_deletion=True) return OStream(sha, type, size, stream) def has_object(self, sha): @@ -199,7 +199,7 @@ def store(self, istream): else: # write object with header, we have to make a new one write_object(istream.type, istream.size, istream.read, writer.write, - chunk_size=self.stream_chunk_size) + chunk_size=self.stream_chunk_size) # END handle direct stream copies finally: if tmp_path: diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 1aa0d511f..595dbf4fc 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -28,7 +28,9 @@ __all__ = ("MemoryDB", ) + class MemoryDB(ObjectDBR, ObjectDBW): + """A memory database stores everything to memory, providing fast IO and object retrieval. It should be used to buffer results and obtain SHAs before writing it to the actual physical storage, as it allows to query whether object already @@ -85,7 +87,6 @@ def sha_iter(self): except AttributeError: return self._cache.keys() - #{ Interface def stream_copy(self, sha_iter, odb): """Copy the streams as identified by sha's yielded by sha_iter into the given odb diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index eaf431a22..6b03d8383 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -31,6 +31,7 @@ class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): + """A database operating on a set of object packs""" # sort the priority list every N queries @@ -113,7 +114,7 @@ def sha_iter(self): def size(self): sizes = [item[1].index().size() for item in self._entities] - return reduce(lambda x,y: x+y, sizes, 0) + return reduce(lambda x, y: x + y, sizes, 0) #} END object db read @@ -127,7 +128,6 @@ def store(self, istream): #} END object db write - #{ Interface def update_cache(self, force=False): @@ -177,7 +177,7 @@ def update_cache(self, force=False): def entities(self): """:return: list of pack entities operated upon by this database""" - return [ item[1] for item in self._entities ] + return [item[1] for item in self._entities] def partial_to_complete_sha(self, partial_binsha, canonical_length): """:return: 20 byte sha as inferred by the given partial binary sha diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index d98912643..83a9f611d 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -8,7 +8,9 @@ __all__ = ('ReferenceDB', ) + class ReferenceDB(CompoundDB): + """A database consisting of database referred to in a file""" # Configuration diff --git a/gitdb/exc.py b/gitdb/exc.py index 73f84d299..d58442f2b 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -5,28 +5,42 @@ """Module with common exceptions""" from gitdb.util import to_hex_sha + class ODBError(Exception): + """All errors thrown by the object database""" + class InvalidDBRoot(ODBError): + """Thrown if an object database cannot be initialized at the given path""" + class BadObject(ODBError): + """The object with the given SHA does not exist. Instantiate with the failed sha""" def __str__(self): return "BadObject: %s" % to_hex_sha(self.args[0]) + class ParseError(ODBError): + """Thrown if the parsing of a file failed due to an invalid format""" + class AmbiguousObjectName(ODBError): + """Thrown if a possibly shortened name does not uniquely represent a single object in the database""" + class BadObjectType(ODBError): + """The object had an unsupported type""" + class UnsupportedOperation(ODBError): + """Thrown if the given operation cannot be supported by the object database""" diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index eb40b44ce..84929ed81 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit eb40b44ce4a6e646aabf7b7091d876738336c42f +Subproject commit 84929ed811142e366d6c5916125302c1419acad6 diff --git a/gitdb/fun.py b/gitdb/fun.py index b7662b495..17da4e5da 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -31,15 +31,15 @@ REF_DELTA = 7 delta_types = (OFS_DELTA, REF_DELTA) -type_id_to_type_map = { - 0 : b'', # EXT 1 - 1 : str_commit_type, - 2 : str_tree_type, - 3 : str_blob_type, - 4 : str_tag_type, - 5 : b'', # EXT 2 - OFS_DELTA : "OFS_DELTA", # OFFSET DELTA - REF_DELTA : "REF_DELTA" # REFERENCE DELTA +type_id_to_type_map = { + 0: b'', # EXT 1 + 1: str_commit_type, + 2: str_tree_type, + 3: str_blob_type, + 4: str_tag_type, + 5: b'', # EXT 2 + OFS_DELTA: "OFS_DELTA", # OFFSET DELTA + REF_DELTA: "REF_DELTA" # REFERENCE DELTA } type_to_type_id_map = { @@ -55,8 +55,8 @@ chunk_size = 1000 * mmap.PAGESIZE __all__ = ('is_loose_object', 'loose_object_header_info', 'msb_size', 'pack_object_header_info', - 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', - 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') + 'write_object', 'loose_object_header', 'stream_copy', 'apply_delta_data', + 'is_equal_canonical_sha', 'connect_deltas', 'DeltaChunkList', 'create_pack_object_header') #{ Structures @@ -72,6 +72,7 @@ def _set_delta_rbound(d, size): # MUST NOT DO THIS HERE return d + def _move_delta_lbound(d, bytes): """Move the delta by the given amount of bytes, reducing its size so that its right bound stays static @@ -89,9 +90,11 @@ def _move_delta_lbound(d, bytes): return d + def delta_duplicate(src): return DeltaChunk(src.to, src.ts, src.so, src.data) + def delta_chunk_apply(dc, bbuf, write): """Apply own data to the target buffer :param bbuf: buffer providing source bytes for copy operations @@ -112,15 +115,16 @@ def delta_chunk_apply(dc, bbuf, write): class DeltaChunk(object): + """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" __slots__ = ( - 'to', # start offset in the target buffer in bytes + 'to', # start offset in the target buffer in bytes 'ts', # size of this chunk in the target buffer in bytes 'so', # start offset in the source buffer in bytes or None 'data', # chunk of bytes to be added to the target buffer, # DeltaChunkList to use as base, or None - ) + ) def __init__(self, to, ts, so, data): self.to = to @@ -142,6 +146,7 @@ def has_data(self): #} END interface + def _closest_index(dcl, absofs): """:return: index at which the given absofs should be inserted. The index points to the DeltaChunk with a target buffer absofs that equals or is greater than @@ -160,7 +165,8 @@ def _closest_index(dcl, absofs): lo = mid + 1 # END handle bound # END for each delta absofs - return len(dcl)-1 + return len(dcl) - 1 + def delta_list_apply(dcl, bbuf, write): """Apply the chain's changes and write the final result using the passed @@ -173,6 +179,7 @@ def delta_list_apply(dcl, bbuf, write): delta_chunk_apply(dc, bbuf, write) # END for each dc + def delta_list_slice(dcl, absofs, size, ndcl): """:return: Subsection of this list at the given absolute offset, with the given size in bytes. @@ -209,6 +216,7 @@ def delta_list_slice(dcl, absofs, size, ndcl): class DeltaChunkList(list): + """List with special functionality to deal with DeltaChunks. There are two types of lists we represent. The one was created bottom-up, working towards the latest delta, the other kind was created top-down, working from the @@ -252,16 +260,16 @@ def compress(self): dc = self[i] i += 1 if dc.data is None: - if first_data_index is not None and i-2-first_data_index > 1: - #if first_data_index is not None: + if first_data_index is not None and i - 2 - first_data_index > 1: + # if first_data_index is not None: nd = StringIO() # new data so = self[first_data_index].to # start offset in target buffer - for x in xrange(first_data_index, i-1): + for x in xrange(first_data_index, i - 1): xdc = self[x] nd.write(xdc.data[:xdc.ts]) # END collect data - del(self[first_data_index:i-1]) + del(self[first_data_index:i - 1]) buf = nd.getvalue() self.insert(first_data_index, DeltaChunk(so, len(buf), 0, buf)) @@ -274,10 +282,10 @@ def compress(self): # END skip non-data chunks if first_data_index is None: - first_data_index = i-1 + first_data_index = i - 1 # END iterate list - #if slen_orig != len(self): + # if slen_orig != len(self): # print "INFO: Reduced delta list len to %f %% of former size" % ((float(len(self)) / slen_orig) * 100) return self @@ -288,7 +296,7 @@ def check_integrity(self, target_size=-1): :raise AssertionError: if the size doen't match""" if target_size > -1: assert self[-1].rbound() == target_size - assert reduce(lambda x,y: x+y, (d.ts for d in self), 0) == target_size + assert reduce(lambda x, y: x + y, (d.ts for d in self), 0) == target_size # END target size verification if len(self) < 2: @@ -301,18 +309,19 @@ def check_integrity(self, target_size=-1): assert len(dc.data) >= dc.ts # END for each dc - left = islice(self, 0, len(self)-1) + left = islice(self, 0, len(self) - 1) right = iter(self) right.next() # this is very pythonic - we might have just use index based access here, # but this could actually be faster - for lft,rgt in izip(left, right): + for lft, rgt in izip(left, right): assert lft.rbound() == rgt.to assert lft.to + lft.ts == rgt.to # END for each pair class TopdownDeltaChunkList(DeltaChunkList): + """Represents a list which is generated by feeding its ancestor streams one by one""" __slots__ = tuple() @@ -356,19 +365,19 @@ def connect_with_next_base(self, bdcl): # END update target bounds if len(ccl) == 1: - self[dci-1] = ccl[0] + self[dci - 1] = ccl[0] else: # maybe try to compute the expenses here, and pick the right algorithm # It would normally be faster than copying everything physically though # TODO: Use a deque here, and decide by the index whether to extend # or extend left ! post_dci = self[dci:] - del(self[dci-1:]) # include deletion of dc + del(self[dci - 1:]) # include deletion of dc self.extend(ccl) self.extend(post_dci) slen = len(self) - dci += len(ccl)-1 # deleted dc, added rest + dci += len(ccl) - 1 # deleted dc, added rest # END handle chunk replacement # END for each chunk @@ -391,6 +400,7 @@ def is_loose_object(m): word = (b0 << 8) + b1 return b0 == 0x78 and (word % 31) == 0 + def loose_object_header_info(m): """ :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the @@ -402,6 +412,7 @@ def loose_object_header_info(m): return type_name, int(size) + def pack_object_header_info(data): """ :return: tuple(type_id, uncompressed_size_in_bytes, byte_offset) @@ -430,6 +441,7 @@ def pack_object_header_info(data): # end performance at expense of maintenance ... return (type_id, size, i) + def create_pack_object_header(obj_type, obj_size): """ :return: string defining the pack header comprised of the object type @@ -439,7 +451,7 @@ def create_pack_object_header(obj_type, obj_size): :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte if PY3: - hdr = bytearray() # output string + hdr = bytearray() # output string c = (obj_type << 4) | (obj_size & 0xf) obj_size >>= 4 @@ -447,10 +459,10 @@ def create_pack_object_header(obj_type, obj_size): hdr.append(c | 0x80) c = obj_size & 0x7f obj_size >>= 7 - #END until size is consumed + # END until size is consumed hdr.append(c) else: - hdr = bytes() # output string + hdr = bytes() # output string c = (obj_type << 4) | (obj_size & 0xf) obj_size >>= 4 @@ -458,11 +470,12 @@ def create_pack_object_header(obj_type, obj_size): hdr += chr(c | 0x80) c = obj_size & 0x7f obj_size >>= 7 - #END until size is consumed + # END until size is consumed hdr += chr(c) # end handle interpreter return hdr + def msb_size(data, offset=0): """ :return: tuple(read_bytes, size) read the msb size from the given random @@ -473,8 +486,8 @@ def msb_size(data, offset=0): hit_msb = False if PY3: while i < l: - c = data[i+offset] - size |= (c & 0x7f) << i*7 + c = data[i + offset] + size |= (c & 0x7f) << i * 7 i += 1 if not c & 0x80: hit_msb = True @@ -483,8 +496,8 @@ def msb_size(data, offset=0): # END while in range else: while i < l: - c = ord(data[i+offset]) - size |= (c & 0x7f) << i*7 + c = ord(data[i + offset]) + size |= (c & 0x7f) << i * 7 i += 1 if not c & 0x80: hit_msb = True @@ -494,7 +507,8 @@ def msb_size(data, offset=0): # end performance ... if not hit_msb: raise AssertionError("Could not find terminating MSB byte in data stream") - return i+offset, size + return i + offset, size + def loose_object_header(type, size): """ @@ -502,6 +516,7 @@ def loose_object_header(type, size): followed by the content stream of size 'size'""" return ('%s %i\0' % (force_text(type), size)).encode('ascii') + def write_object(type, size, read, write, chunk_size=chunk_size): """ Write the object as identified by type, size and source_stream into the @@ -522,6 +537,7 @@ def write_object(type, size, read, write, chunk_size=chunk_size): return tbw + def stream_copy(read, write, size, chunk_size): """ Copy a stream up to size bytes using the provided read and write methods, @@ -532,7 +548,7 @@ def stream_copy(read, write, size, chunk_size): # WRITE ALL DATA UP TO SIZE while True: - cs = min(chunk_size, size-dbw) + cs = min(chunk_size, size - dbw) # NOTE: not all write methods return the amount of written bytes, like # mmap.write. Its bad, but we just deal with it ... perhaps its not # even less efficient @@ -548,6 +564,7 @@ def stream_copy(read, write, size, chunk_size): # END duplicate data return dbw + def connect_deltas(dstreams): """ Read the condensed delta chunk information from dstream and merge its information @@ -602,7 +619,7 @@ def connect_deltas(dstreams): rbound = cp_off + cp_size if (rbound < cp_size or - rbound > base_size): + rbound > base_size): break dcl.append(DeltaChunk(tbw, cp_size, cp_off, None)) @@ -610,7 +627,7 @@ def connect_deltas(dstreams): elif c: # NOTE: in C, the data chunks should probably be concatenated here. # In python, we do it as a post-process - dcl.append(DeltaChunk(tbw, c, 0, db[i:i+c])) + dcl.append(DeltaChunk(tbw, c, 0, db[i:i + c])) i += c tbw += c else: @@ -632,6 +649,7 @@ def connect_deltas(dstreams): return tdcl + def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): """ Apply data from a delta buffer using a source buffer to the target file @@ -678,11 +696,11 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): rbound = cp_off + cp_size if (rbound < cp_size or - rbound > src_buf_size): + rbound > src_buf_size): break write(buffer(src_buf, cp_off, cp_size)) elif c: - write(db[i:i+c]) + write(db[i:i + c]) i += c else: raise ValueError("unexpected delta opcode 0") @@ -721,11 +739,11 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): rbound = cp_off + cp_size if (rbound < cp_size or - rbound > src_buf_size): + rbound > src_buf_size): break write(buffer(src_buf, cp_off, cp_size)) elif c: - write(db[i:i+c]) + write(db[i:i + c]) i += c else: raise ValueError("unexpected delta opcode 0") @@ -749,7 +767,7 @@ def is_equal_canonical_sha(canonical_length, match, sha1): return False if canonical_length - binary_length and \ - (byte_ord(match[-1]) ^ byte_ord(sha1[len(match)-1])) & 0xf0: + (byte_ord(match[-1]) ^ byte_ord(sha1[len(match) - 1])) & 0xf0: return False # END handle uneven canonnical length return True diff --git a/gitdb/pack.py b/gitdb/pack.py index 375cc59a9..b4ba7876c 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -72,8 +72,6 @@ __all__ = ('PackIndexFile', 'PackFile', 'PackEntity') - - #{ Utilities def pack_object_at(cursor, offset, as_stream): @@ -107,7 +105,7 @@ def pack_object_at(cursor, offset, as_stream): total_rela_offset = i # REF DELTA elif type_id == REF_DELTA: - total_rela_offset = data_rela_offset+20 + total_rela_offset = data_rela_offset + 20 delta_info = data[data_rela_offset:total_rela_offset] # BASE OBJECT else: @@ -129,6 +127,7 @@ def pack_object_at(cursor, offset, as_stream): # END handle info # END handle stream + def write_stream_to_pack(read, write, zstream, base_crc=None): """Copy a stream as read from read function, zip it, and write the result. Count the number of written bytes and return it @@ -142,7 +141,7 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): crc = 0 if want_crc: crc = base_crc - #END initialize crc + # END initialize crc while True: chunk = read(chunk_size) @@ -153,18 +152,18 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): if want_crc: crc = crc32(compressed, crc) - #END handle crc + # END handle crc if len(chunk) != chunk_size: break - #END copy loop + # END copy loop compressed = zstream.flush() bw += len(compressed) write(compressed) if want_crc: crc = crc32(compressed, crc) - #END handle crc + # END handle crc return (br, bw, crc) @@ -173,6 +172,7 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): class IndexWriter(object): + """Utility to cache index information, allowing to write all information later in one go to the given stream **Note:** currently only writes v2 indices""" @@ -198,15 +198,15 @@ def write(self, pack_sha, write): sha_write(pack(">L", PackIndexFile.index_version_default)) # fanout - tmplist = list((0,)*256) # fanout or list with 64 bit offsets + tmplist = list((0,) * 256) # fanout or list with 64 bit offsets for t in self._objs: tmplist[byte_ord(t[0][0])] += 1 - #END prepare fanout + # END prepare fanout for i in xrange(255): v = tmplist[i] sha_write(pack('>L', v)) - tmplist[i+1] += v - #END write each fanout entry + tmplist[i + 1] += v + # END write each fanout entry sha_write(pack('>L', tmplist[255])) # sha1 ordered @@ -215,8 +215,8 @@ def write(self, pack_sha, write): # crc32 for t in self._objs: - sha_write(pack('>L', t[1]&0xffffffff)) - #END for each crc + sha_write(pack('>L', t[1] & 0xffffffff)) + # END for each crc tmplist = list() # offset 32 @@ -224,15 +224,15 @@ def write(self, pack_sha, write): ofs = t[2] if ofs > 0x7fffffff: tmplist.append(ofs) - ofs = 0x80000000 + len(tmplist)-1 - #END hande 64 bit offsets - sha_write(pack('>L', ofs&0xffffffff)) - #END for each offset + ofs = 0x80000000 + len(tmplist) - 1 + # END hande 64 bit offsets + sha_write(pack('>L', ofs & 0xffffffff)) + # END for each offset # offset 64 for ofs in tmplist: sha_write(pack(">Q", ofs)) - #END for each offset + # END for each offset # trailer assert(len(pack_sha) == 20) @@ -242,8 +242,8 @@ def write(self, pack_sha, write): return sha - class PackIndexFile(LazyMixin): + """A pack index provides offsets into the corresponding pack, allowing to find locations for offsets faster.""" @@ -273,8 +273,9 @@ def _set_cache_(self, attr): self._cursor = mman.make_cursor(self._indexpath).use_region() # We will assume that the index will always fully fit into memory ! if mman.window_size() > 0 and self._cursor.file_size() > mman.window_size(): - raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % (self._indexpath, self._cursor.file_size(), mman.window_size())) - #END assert window size + raise AssertionError("The index file at %s is too large to fit into a mapped window (%i > %i). This is a limitation of the implementation" % ( + self._indexpath, self._cursor.file_size(), mman.window_size())) + # END assert window size else: # now its time to initialize everything - if we are here, someone wants # to access the fanout table or related properties @@ -293,27 +294,25 @@ def _set_cache_(self, attr): setattr(self, fname, getattr(self, "_%s_v%i" % (fname, self._version))) # END for each function to initialize - # INITIALIZE DATA # byte offset is 8 if version is 2, 0 otherwise self._initialize() # END handle attributes - #{ Access V1 def _entry_v1(self, i): """:return: tuple(offset, binsha, 0)""" - return unpack_from(">L20s", self._cursor.map(), 1024 + i*24) + (0, ) + return unpack_from(">L20s", self._cursor.map(), 1024 + i * 24) + (0, ) def _offset_v1(self, i): """see ``_offset_v2``""" - return unpack_from(">L", self._cursor.map(), 1024 + i*24)[0] + return unpack_from(">L", self._cursor.map(), 1024 + i * 24)[0] def _sha_v1(self, i): """see ``_sha_v2``""" - base = 1024 + (i*24)+4 - return self._cursor.map()[base:base+20] + base = 1024 + (i * 24) + 4 + return self._cursor.map()[base:base + 20] def _crc_v1(self, i): """unsupported""" @@ -343,7 +342,7 @@ def _offset_v2(self, i): def _sha_v2(self, i): """:return: sha at the given index of this file index instance""" base = self._sha_list_offset + i * 20 - return self._cursor.map()[base:base+20] + return self._cursor.map()[base:base + 20] def _crc_v2(self, i): """:return: 4 bytes crc for the object at index i""" @@ -369,7 +368,7 @@ def _read_fanout(self, byte_offset): out = list() append = out.append for i in xrange(256): - append(unpack_from('>L', d, byte_offset + i*4)[0]) + append(unpack_from('>L', d, byte_offset + i * 4)[0]) # END for each entry return out @@ -421,7 +420,7 @@ def sha_to_index(self, sha): get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: - lo = self._fanout_table[first_byte-1] + lo = self._fanout_table[first_byte - 1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection # bisect until we have the sha @@ -455,7 +454,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): get_sha = self.sha lo = 0 # lower index, the left bound of the bisection if first_byte != 0: - lo = self._fanout_table[first_byte-1] + lo = self._fanout_table[first_byte - 1] hi = self._fanout_table[first_byte] # the upper, right bound of the bisection # fill the partial to full 20 bytes @@ -481,7 +480,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): if is_equal_canonical_sha(canonical_length, partial_bin_sha, cur_sha): next_sha = None if lo + 1 < self.size(): - next_sha = get_sha(lo+1) + next_sha = get_sha(lo + 1) if next_sha and next_sha == cur_sha: raise AmbiguousObjectName(partial_bin_sha) return lo @@ -500,6 +499,7 @@ def sha_to_index(self, sha): class PackFile(LazyMixin): + """A pack is a file written according to the Version 2 for git packs As we currently use memory maps, it could be assumed that the maximum size of @@ -516,7 +516,7 @@ class PackFile(LazyMixin): pack_version_default = 2 # offset into our data at which the first object starts - first_object_offset = 3*4 # header bytes + first_object_offset = 3 * 4 # header bytes footer_size = 20 # final sha def __init__(self, packpath): @@ -549,7 +549,6 @@ def _iter_objects(self, start_offset, as_stream=True): stream_copy(ostream.read, null.write, ostream.size, chunk_size) cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() - # if a stream is requested, reset it beforehand # Otherwise return the Stream object directly, its derived from the # info object @@ -578,7 +577,7 @@ def data(self): def checksum(self): """:return: 20 byte sha1 hash on all object sha's contained in this file""" - return self._cursor.use_region(self._cursor.file_size()-20).buffer()[:] + return self._cursor.use_region(self._cursor.file_size() - 20).buffer()[:] def path(self): """:return: path to the packfile""" @@ -645,13 +644,14 @@ def stream_iter(self, start_offset=0): class PackEntity(LazyMixin): + """Combines the PackIndexFile and the PackFile into one, allowing the actual objects to be resolved and iterated""" - __slots__ = ( '_index', # our index file - '_pack', # our pack file - '_offset_map' # on demand dict mapping one offset to the next consecutive one - ) + __slots__ = ('_index', # our index file + '_pack', # our pack file + '_offset_map' # on demand dict mapping one offset to the next consecutive one + ) IndexFileCls = PackIndexFile PackFileCls = PackFile @@ -673,7 +673,7 @@ def _set_cache_(self, attr): offset_map = None if len(offsets_sorted) == 1: - offset_map = { offsets_sorted[0] : last_offset } + offset_map = {offsets_sorted[0]: last_offset} else: iter_offsets = iter(offsets_sorted) iter_offsets_plus_one = iter(offsets_sorted) @@ -895,10 +895,9 @@ def collect_streams(self, sha): :raise BadObject:""" return self.collect_streams_at_offset(self._index.offset(self._sha_to_index(sha))) - @classmethod def write_pack(cls, object_iter, pack_write, index_write=None, - object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + object_count=None, zlib_compression=zlib.Z_BEST_SPEED): """ Create a new pack by putting all objects obtained by the object_iterator into a pack which is written using the pack_write method. @@ -923,9 +922,9 @@ def write_pack(cls, object_iter, pack_write, index_write=None, if not object_count: if not isinstance(object_iter, (tuple, list)): objs = list(object_iter) - #END handle list type + # END handle list type object_count = len(objs) - #END handle object + # END handle object pack_writer = FlexibleSha1Writer(pack_write) pwrite = pack_writer.write @@ -939,7 +938,7 @@ def write_pack(cls, object_iter, pack_write, index_write=None, if wants_index: index = IndexWriter() - #END handle index header + # END handle index header actual_count = 0 for obj in objs: @@ -952,30 +951,31 @@ def write_pack(cls, object_iter, pack_write, index_write=None, crc = crc32(hdr) else: crc = None - #END handle crc + # END handle crc pwrite(hdr) # data stream zstream = zlib.compressobj(zlib_compression) ostream = obj.stream - br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc = crc) + br, bw, crc = write_stream_to_pack(ostream.read, pwrite, zstream, base_crc=crc) assert(br == obj.size) if wants_index: index.append(obj.binsha, crc, ofs) - #END handle index + # END handle index ofs += len(hdr) + bw if actual_count == object_count: break - #END abort once we are done - #END for each object + # END abort once we are done + # END for each object if actual_count != object_count: - raise ValueError("Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) - #END count assertion + raise ValueError( + "Expected to write %i objects into pack, but received only %i from iterators" % (object_count, actual_count)) + # END count assertion # write footer - pack_sha = pack_writer.sha(as_hex = False) + pack_sha = pack_writer.sha(as_hex=False) assert len(pack_sha) == 20 pack_write(pack_sha) ofs += len(pack_sha) # just for completeness ;) @@ -983,12 +983,12 @@ def write_pack(cls, object_iter, pack_write, index_write=None, index_sha = None if wants_index: index_sha = index.write(pack_sha, index_write) - #END handle index + # END handle index return pack_sha, index_sha @classmethod - def create(cls, object_iter, base_dir, object_count = None, zlib_compression = zlib.Z_BEST_SPEED): + def create(cls, object_iter, base_dir, object_count=None, zlib_compression=zlib.Z_BEST_SPEED): """Create a new on-disk entity comprised of a properly named pack file and a properly named and corresponding index file. The pack contains all OStream objects contained in object iter. :param base_dir: directory which is to contain the files @@ -1012,5 +1012,4 @@ def create(cls, object_iter, base_dir, object_count = None, zlib_compression = z return cls(new_pack_path) - #} END interface diff --git a/gitdb/stream.py b/gitdb/stream.py index b0a89002a..4478a0f44 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -38,14 +38,15 @@ except ImportError: pass -__all__ = ( 'DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', - 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', - 'FDStream', 'NullStream') +__all__ = ('DecompressMemMapReader', 'FDCompressedSha1Writer', 'DeltaApplyReader', + 'Sha1Writer', 'FlexibleSha1Writer', 'ZippedStoreShaWriter', 'FDCompressedSha1Writer', + 'FDStream', 'NullStream') #{ RO Streams class DecompressMemMapReader(LazyMixin): + """Reads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand buffered decompression accordingly @@ -63,9 +64,9 @@ class DecompressMemMapReader(LazyMixin): to better support streamed reading - it would only need to keep the mmap and decompress it into chunks, thats all ... """ __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', - '_cbr', '_phi') + '_cbr', '_phi') - max_read_size = 512*1024 # currently unused + max_read_size = 512 * 1024 # currently unused def __init__(self, m, close_on_deletion, size=None): """Initialize with mmap for stream reading @@ -214,7 +215,6 @@ def read(self, size=-1): return bytes() # END handle depletion - # deplete the buffer, then just continue using the decompress object # which has an own buffer. We just need this to transparently parse the # header from the zlib stream @@ -263,7 +263,6 @@ def read(self, size=-1): self._cwe = cws + size # END handle tail - # if window is too small, make it larger so zip can decompress something if self._cwe - self._cws < 8: self._cwe = self._cws + 8 @@ -285,7 +284,7 @@ def read(self, size=-1): unused_datalen = len(self._zip.unconsumed_tail) else: unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) - # end handle very special case ... + # end handle very special case ... self._cbr += len(indata) - unused_datalen self._br += len(dcompdat) @@ -301,12 +300,13 @@ def read(self, size=-1): # to read, if we are called by compressed_bytes_read - it manipulates # us to empty the stream if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s: - dcompdat += self.read(size-len(dcompdat)) + dcompdat += self.read(size - len(dcompdat)) # END handle special case return dcompdat class DeltaApplyReader(LazyMixin): + """A reader which dynamically applies pack deltas to a base object, keeping the memory demands to a minimum. @@ -332,15 +332,15 @@ class DeltaApplyReader(LazyMixin): * cmd == 0 - invalid operation ( or error in delta stream ) """ __slots__ = ( - "_bstream", # base stream to which to apply the deltas - "_dstreams", # tuple of delta stream readers - "_mm_target", # memory map of the delta-applied data - "_size", # actual number of bytes in _mm_target - "_br" # number of bytes read - ) + "_bstream", # base stream to which to apply the deltas + "_dstreams", # tuple of delta stream readers + "_mm_target", # memory map of the delta-applied data + "_size", # actual number of bytes in _mm_target + "_br" # number of bytes read + ) #{ Configuration - k_max_memory_move = 250*1000*1000 + k_max_memory_move = 250 * 1000 * 1000 #} END configuration def __init__(self, stream_list): @@ -414,7 +414,6 @@ def _set_cache_brute_(self, attr): base_size = target_size = max(base_size, max_target_size) # END adjust buffer sizes - # Allocate private memory map big enough to hold the first base buffer # We need random access to it bbuf = allocate_memory(base_size) @@ -440,11 +439,11 @@ def _set_cache_brute_(self, attr): ddata = allocate_memory(dstream.size - offset) ddata.write(dbuf) # read the rest from the stream. The size we give is larger than necessary - stream_copy(dstream.read, ddata.write, dstream.size, 256*mmap.PAGESIZE) + stream_copy(dstream.read, ddata.write, dstream.size, 256 * mmap.PAGESIZE) ####################################################################### if 'c_apply_delta' in globals(): - c_apply_delta(bbuf, ddata, tbuf); + c_apply_delta(bbuf, ddata, tbuf) else: apply_delta_data(bbuf, src_size, ddata, len(ddata), tbuf.write) ####################################################################### @@ -463,7 +462,6 @@ def _set_cache_brute_(self, attr): self._mm_target = bbuf self._size = final_target_size - #{ Configuration if not has_perf_mod: _set_cache_ = _set_cache_brute_ @@ -512,13 +510,13 @@ def new(cls, stream_list): # END single object special handling if stream_list[-1].type_id in delta_types: - raise ValueError("Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) + raise ValueError( + "Cannot resolve deltas if there is no base object stream, last one was type: %s" % stream_list[-1].type) # END check stream return cls(stream_list) #} END interface - #{ OInfo like Interface @property @@ -543,6 +541,7 @@ def size(self): #{ W Streams class Sha1Writer(object): + """Simple stream writer which produces a sha whenever you like as it degests everything it is supposed to write""" __slots__ = "sha1" @@ -565,7 +564,7 @@ def write(self, data): #{ Interface - def sha(self, as_hex = False): + def sha(self, as_hex=False): """:return: sha so far :param as_hex: if True, sha will be hex-encoded, binary otherwise""" if as_hex: @@ -576,6 +575,7 @@ def sha(self, as_hex = False): class FlexibleSha1Writer(Sha1Writer): + """Writer producing a sha1 while passing on the written bytes to the given write function""" __slots__ = 'writer' @@ -590,8 +590,10 @@ def write(self, data): class ZippedStoreShaWriter(Sha1Writer): + """Remembers everything someone writes to it and generates a sha""" __slots__ = ('buf', 'zip') + def __init__(self): Sha1Writer.__init__(self) self.buf = BytesIO() @@ -623,6 +625,7 @@ def getvalue(self): class FDCompressedSha1Writer(Sha1Writer): + """Digests data written to it, making the sha available, then compress the data and write it to the file descriptor @@ -662,10 +665,12 @@ def close(self): class FDStream(object): + """A simple wrapper providing the most basic functions on a file descriptor with the fileobject interface. Cannot use os.fdopen as the resulting stream takes ownership""" __slots__ = ("_fd", '_pos') + def __init__(self, fd): self._fd = fd self._pos = 0 @@ -694,6 +699,7 @@ def close(self): class NullStream(object): + """A stream that does nothing but providing a stream interface. Use it like /dev/null""" __slots__ = tuple() diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index af6d9e0fd..528bcc144 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -32,7 +32,9 @@ __all__ = ('TestDBBase', 'with_rw_directory', 'with_packs_rw', 'fixture_path') + class TestDBBase(TestBase): + """Base class providing testing routines on databases""" # data @@ -65,7 +67,6 @@ def _assert_object_writing_simple(self, db): assert len(shas) == db.size() assert len(shas[0]) == 20 - def _assert_object_writing(self, db): """General tests to verify object writing, compatible to ObjectDBW **Note:** requires write access to the database""" @@ -126,4 +127,3 @@ def _assert_object_writing(self, db): assert ostream.getvalue() == new_ostream.getvalue() # END for each data set # END for each dry_run mode - diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index e141c2ba0..f96206744 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -3,7 +3,7 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from gitdb.test.db.lib import ( - TestDBBase, + TestDBBase, fixture_path, with_rw_directory ) @@ -12,6 +12,7 @@ from gitdb.base import OStream, OInfo from gitdb.util import hex_to_bin, bin_to_hex + class TestGitDB(TestDBBase): def test_reading(self): @@ -28,8 +29,7 @@ def test_reading(self): assert gdb.size() >= ni sha_list = list(gdb.sha_iter()) assert len(sha_list) == gdb.size() - sha_list = sha_list[:ni] # speed up tests ... - + sha_list = sha_list[:ni] # speed up tests ... # This is actually a test for compound functionality, but it doesn't # have a separate test module @@ -39,7 +39,7 @@ def test_reading(self): # mix even/uneven hexshas for i, binsha in enumerate(sha_list): - assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8-(i%2)]) == binsha + assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8 - (i % 2)]) == binsha # END for each sha self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 1d6af9c99..024c194a2 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -3,13 +3,14 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from gitdb.test.db.lib import ( - TestDBBase, + TestDBBase, with_rw_directory ) from gitdb.db import LooseObjectDB from gitdb.exc import BadObject from gitdb.util import bin_to_hex + class TestLooseDB(TestDBBase): @with_rw_directory diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index 97f721719..eb563c098 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -11,6 +11,7 @@ LooseObjectDB ) + class TestMemoryDB(TestDBBase): @with_rw_directory diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 963a71af7..a90158151 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -14,6 +14,7 @@ import os import random + class TestPackDB(TestDBBase): @with_rw_directory @@ -53,7 +54,6 @@ def test_writing(self, path): pdb.stream(sha) # END for each sha to query - # test short finding - be a bit more brutal here max_bytes = 19 min_bytes = 2 @@ -61,10 +61,10 @@ def test_writing(self, path): for i, sha in enumerate(sha_list): short_sha = sha[:max((i % max_bytes), min_bytes)] try: - assert pdb.partial_to_complete_sha(short_sha, len(short_sha)*2) == sha + assert pdb.partial_to_complete_sha(short_sha, len(short_sha) * 2) == sha except AmbiguousObjectName: num_ambiguous += 1 - pass # valid, we can have short objects + pass # valid, we can have short objects # END exception handling # END for each sha to find diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index db930827b..b774bafe4 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -3,8 +3,8 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php from gitdb.test.db.lib import ( - TestDBBase, - with_rw_directory, + TestDBBase, + with_rw_directory, fixture_path ) from gitdb.db import ReferenceDB @@ -16,6 +16,7 @@ import os + class TestReferenceDB(TestDBBase): def make_alt_file(self, alt_path, alt_list): diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index d09b1cb8e..c4acd9218 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -24,6 +24,7 @@ #{ Bases class TestBase(unittest.TestCase): + """Base class for all tests""" @@ -49,6 +50,7 @@ def wrapper(self, *args, **kwargs): def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the test suceeds, but leave it otherwise to aid additional debugging""" + def wrapper(self): path = tempfile.mktemp(prefix=func.__name__) os.mkdir(path) @@ -78,6 +80,7 @@ def wrapper(self): def with_packs_rw(func): """Function that provides a path into which the packs for testing should be copied. Will pass on the path to the actual function afterwards""" + def wrapper(self, path): src_pack_glob = fixture_path('packs/*') copy_files_globbed(src_pack_glob, path, hard_link_ok=True) @@ -91,12 +94,14 @@ def wrapper(self, path): #{ Routines + def fixture_path(relapath=''): """:return: absolute path into the fixture directory :param relapath: relative path into the fixtures directory, or '' to obtain the fixture directory itself""" return os.path.join(os.path.dirname(__file__), 'fixtures', relapath) + def copy_files_globbed(source_glob, target_dir, hard_link_ok=False): """Copy all files found according to the given source glob into the target directory :param hard_link_ok: if True, hard links will be created if possible. Otherwise @@ -127,11 +132,13 @@ def make_bytes(size_in_bytes, randomize=False): a = array('i', producer) return a.tostring() + def make_object(type, data): """:return: bytes resembling an uncompressed object""" odata = "blob %i\0" % len(data) return odata.encode("ascii") + data + def make_memory_file(size_in_bytes, randomize=False): """:return: tuple(size_of_stream, stream) :param randomize: try to produce a very random stream""" @@ -142,24 +149,27 @@ def make_memory_file(size_in_bytes, randomize=False): #{ Stream Utilities + class DummyStream(object): - def __init__(self): - self.was_read = False - self.bytes = 0 - self.closed = False - def read(self, size): - self.was_read = True - self.bytes = size + def __init__(self): + self.was_read = False + self.bytes = 0 + self.closed = False - def close(self): - self.closed = True + def read(self, size): + self.was_read = True + self.bytes = size - def _assert(self): - assert self.was_read + def close(self): + self.closed = True + + def _assert(self): + assert self.was_read class DeriveTest(OStream): + def __init__(self, sha, type, size, stream, *args, **kwargs): self.myarg = kwargs.pop('myarg') self.args = args diff --git a/gitdb/test/performance/__init__.py b/gitdb/test/performance/__init__.py index 8b1378917..e69de29bb 100644 --- a/gitdb/test/performance/__init__.py +++ b/gitdb/test/performance/__init__.py @@ -1 +0,0 @@ - diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index ec45cf3a7..cbc52bc77 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -13,22 +13,17 @@ #} END invariants - -#{ Base Classes +#{ Base Classes class TestBigRepoR(TestBase): + """TestCase providing access to readonly 'big' repositories using the following member variables: - + * gitrepopath - + * read-only base path of the git source repository, i.e. .../git/.git""" - - #{ Invariants - head_sha_2k = '235d521da60e4699e5bd59ac658b5b48bd76ddca' - head_sha_50 = '32347c375250fd470973a5d76185cac718955fd5' - #} END invariants - + def setUp(self): try: super(TestBigRepoR, self).setUp() @@ -37,11 +32,12 @@ def setUp(self): self.gitrepopath = os.environ.get(k_env_git_repo) if not self.gitrepopath: - logging.info("You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository") + logging.info( + "You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository", k_env_git_repo) ospd = os.path.dirname self.gitrepopath = os.path.join(ospd(ospd(ospd(ospd(__file__)))), '.git') # end assure gitrepo is set assert self.gitrepopath.endswith('.git') - + #} END base classes diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index e46031115..bdd2b0a37 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -6,7 +6,7 @@ from __future__ import print_function from gitdb.test.performance.lib import ( - TestBigRepoR + TestBigRepoR ) from gitdb import ( @@ -24,19 +24,20 @@ import os from time import time + class TestPackedDBPerformance(TestBigRepoR): - @skip_on_travis_ci + @skip_on_travis_ci def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - + # sha lookup st = time() sha_list = list(pdb.sha_iter()) elapsed = time() - st ns = len(sha_list) print("PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed), file=sys.stderr) - + # sha lookup: best-case and worst case access pdb_pack_info = pdb._pack_info # END shuffle shas @@ -45,13 +46,14 @@ def test_pack_random_access(self): pdb_pack_info(sha) # END for each sha to look up elapsed = time() - st - + # discard cache del(pdb._entities) pdb.entities() - print("PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % (ns, len(pdb.entities()), elapsed, ns / elapsed), file=sys.stderr) + print("PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % + (ns, len(pdb.entities()), elapsed, ns / elapsed), file=sys.stderr) # END for each random mode - + # query info and streams only max_items = 10000 # can wait longer when testing memory for pdb_fun in (pdb.info, pdb.stream): @@ -59,9 +61,10 @@ def test_pack_random_access(self): for sha in sha_list[:max_items]: pdb_fun(sha) elapsed = time() - st - print("PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed), file=sys.stderr) + print("PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % + (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed), file=sys.stderr) # END for each function - + # retrieve stream and read all max_items = 5000 pdb_stream = pdb.stream @@ -74,8 +77,9 @@ def test_pack_random_access(self): total_size += stream.size elapsed = time() - st total_kib = total_size / 1000 - print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed), file=sys.stderr) - + print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % + (max_items, total_kib, total_kib / elapsed, elapsed, max_items / elapsed), file=sys.stderr) + @skip_on_travis_ci def test_loose_correctness(self): """based on the pack(s) of our packed object DB, we will just copy and verify all objects in the back @@ -89,7 +93,7 @@ def test_loose_correctness(self): mdb = MemoryDB() for c, sha in enumerate(pdb.sha_iter()): ostream = pdb.stream(sha) - # the issue only showed on larger files which are hardly compressible ... + # the issue only showed on larger files which are hardly compressible ... if ostream.type != str_blob_type: continue istream = IStream(ostream.type, ostream.size, ostream.stream) @@ -101,7 +105,7 @@ def test_loose_correctness(self): if c and c % 1000 == 0: print("Verified %i loose object compression/decompression cycles" % c, file=sys.stderr) mdb._cache.clear() - # end for each sha to copy + # end for each sha to copy @skip_on_travis_ci def test_correctness(self): @@ -124,6 +128,6 @@ def test_correctness(self): # END for each index # END for each entity elapsed = time() - st - print("PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed), file=sys.stderr) + print("PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % + (count, crc, elapsed, count / elapsed), file=sys.stderr) # END for each verify mode - diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index fe160ea54..f805e59fa 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -6,7 +6,7 @@ from __future__ import print_function from gitdb.test.performance.lib import ( - TestBigRepoR + TestBigRepoR ) from gitdb.db.pack import PackedDB @@ -18,27 +18,29 @@ import sys from time import time + class CountedNullStream(NullStream): __slots__ = '_bw' + def __init__(self): self._bw = 0 - + def bytes_written(self): return self._bw - + def write(self, d): self._bw += NullStream.write(self, d) - + class TestPackStreamingPerformance(TestBigRepoR): - + @skip_on_travis_ci def test_pack_writing(self): # see how fast we can write a pack from object streams. # This will not be fast, as we take time for decompressing the streams as well ostream = CountedNullStream() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - + ni = 1000 count = 0 st = time() @@ -47,22 +49,23 @@ def test_pack_writing(self): pdb.stream(sha) if count == ni: break - #END gather objects for pack-writing + # END gather objects for pack-writing elapsed = time() - st - print("PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % (ni, elapsed, ni / elapsed), file=sys.stderr) - + print("PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % + (ni, elapsed, ni / elapsed), file=sys.stderr) + st = time() PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) elapsed = time() - st total_kb = ostream.bytes_written() / 1000 - print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb/elapsed), sys.stderr) - - + print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % + (total_kb, elapsed, total_kb / elapsed), sys.stderr) + @skip_on_travis_ci def test_stream_reading(self): # raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) - + # streaming only, meant for --with-profile runs ni = 5000 count = 0 @@ -78,5 +81,5 @@ def test_stream_reading(self): count += 1 elapsed = time() - st total_kib = total_size / 1000 - print(sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (ni, total_kib, total_kib/elapsed , elapsed, ni / elapsed), sys.stderr) - + print(sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % + (ni, total_kib, total_kib / elapsed, elapsed, ni / elapsed), sys.stderr) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 84c9dea3f..bd66b26ad 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -35,22 +35,22 @@ def read_chunked_stream(stream): # END read stream loop assert total == stream.size return stream - - + + #} END utilities class TestObjDBPerformance(TestBigRepoR): - - large_data_size_bytes = 1000*1000*50 # some MiB should do it - moderate_data_size_bytes = 1000*1000*1 # just 1 MiB - - @skip_on_travis_ci + + large_data_size_bytes = 1000 * 1000 * 50 # some MiB should do it + moderate_data_size_bytes = 1000 * 1000 * 1 # just 1 MiB + + @skip_on_travis_ci @with_rw_directory def test_large_data_streaming(self, path): ldb = LooseObjectDB(path) string_ios = list() # list of streams we previously created - - # serial mode + + # serial mode for randomize in range(2): desc = (randomize and 'random ') or '' print("Creating %s data ..." % desc, file=sys.stderr) @@ -59,32 +59,32 @@ def test_large_data_streaming(self, path): elapsed = time() - st print("Done (in %f s)" % elapsed, file=sys.stderr) string_ios.append(stream) - - # writing - due to the compression it will seem faster than it is + + # writing - due to the compression it will seem faster than it is st = time() sha = ldb.store(IStream('blob', size, stream)).binsha elapsed_add = time() - st assert ldb.has_object(sha) db_file = ldb.readable_db_object_path(bin_to_hex(sha)) fsize_kib = os.path.getsize(db_file) / 1000 - - + size_kib = size / 1000 - print("Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add), file=sys.stderr) - + print("Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % + (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add), file=sys.stderr) + # reading all at once st = time() ostream = ldb.stream(sha) shadata = ostream.read() elapsed_readall = time() - st - + stream.seek(0) assert shadata == stream.getvalue() - print("Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, elapsed_readall, size_kib / elapsed_readall), file=sys.stderr) - - + print("Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % + (size_kib, desc, elapsed_readall, size_kib / elapsed_readall), file=sys.stderr) + # reading in chunks of 1 MiB - cs = 512*1000 + cs = 512 * 1000 chunks = list() st = time() ostream = ldb.stream(sha) @@ -95,13 +95,14 @@ def test_large_data_streaming(self, path): break # END read in chunks elapsed_readchunks = time() - st - + stream.seek(0) assert b''.join(chunks) == stream.getvalue() - + cs_kib = cs / 1000 - print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks), file=sys.stderr) - + print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % + (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks), file=sys.stderr) + # del db file so we keep something to do os.remove(db_file) # END for each randomization factor diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 578c29f73..519cdfdc1 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -4,10 +4,10 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Test for object db""" from gitdb.test.lib import ( - TestBase, - DummyStream, - DeriveTest, - ) + TestBase, + DummyStream, + DeriveTest, +) from gitdb import ( OInfo, @@ -20,11 +20,11 @@ ) from gitdb.util import ( NULL_BIN_SHA - ) +) from gitdb.typ import ( str_blob_type - ) +) class TestBaseTypes(TestBase): @@ -54,7 +54,6 @@ def test_streams(self): assert dpinfo.delta_info == sha assert dpinfo.pack_offset == 0 - # test ostream stream = DummyStream() ostream = OStream(*(info + (stream, ))) @@ -80,7 +79,7 @@ def test_streams(self): assert stream.bytes == 5 # derive with own args - DeriveTest(sha, str_blob_type, s, stream, 'mine',myarg = 3)._assert() + DeriveTest(sha, str_blob_type, s, stream, 'mine', myarg=3)._assert() # test istream istream = IStream(str_blob_type, s, stream) diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index aa43a093f..ed0a885bf 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -4,7 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with examples from the tutorial section of the docs""" from gitdb.test.lib import ( - TestBase, + TestBase, fixture_path ) from gitdb import IStream @@ -12,6 +12,7 @@ from io import BytesIO + class TestExamples(TestBase): def test_base(self): diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 3ab2fec07..ff1057237 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -43,6 +43,7 @@ def bin_sha_from_filename(filename): return to_bin_sha(os.path.splitext(os.path.basename(filename))[0][5:]) #} END utilities + class TestPack(TestBase): packindexfile_v1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx'), 1, 67) @@ -50,8 +51,8 @@ class TestPack(TestBase): packindexfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx'), 2, 42) packfile_v2_1 = (fixture_path('packs/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack'), 2, packindexfile_v1[2]) packfile_v2_2 = (fixture_path('packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack'), 2, packindexfile_v2[2]) - packfile_v2_3_ascii = (fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) - + packfile_v2_3_ascii = ( + fixture_path('packs/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack'), 2, packindexfile_v2_3_ascii[2]) def _assert_index_file(self, index, version, size): assert index.packfile_checksum() != index.indexfile_checksum() @@ -74,13 +75,12 @@ def _assert_index_file(self, index, version, size): assert entry[2] == index.crc(oidx) # verify partial sha - for l in (4,8,11,17,20): - assert index.partial_sha_to_index(sha[:l], l*2) == oidx + for l in (4, 8, 11, 17, 20): + assert index.partial_sha_to_index(sha[:l], l * 2) == oidx # END for each object index in indexfile self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) - def _assert_pack_file(self, pack, version, size): assert pack.version() == 2 assert pack.size() == size @@ -120,7 +120,6 @@ def _assert_pack_file(self, pack, version, size): dstream.seek(0) assert dstream.read() == data - # read chunks # NOTE: the current implementation is safe, it basically transfers # all calls to the underlying memory map @@ -128,7 +127,6 @@ def _assert_pack_file(self, pack, version, size): # END for each object assert num_obj == size - def test_pack_index(self): # check version 1 and 2 for indexfile, version, size in (self.packindexfile_v1, self.packindexfile_v2): @@ -146,9 +144,9 @@ def test_pack(self): @with_rw_directory def test_pack_entity(self, rw_dir): pack_objs = list() - for packinfo, indexinfo in ( (self.packfile_v2_1, self.packindexfile_v1), - (self.packfile_v2_2, self.packindexfile_v2), - (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): + for packinfo, indexinfo in ((self.packfile_v2_1, self.packindexfile_v1), + (self.packfile_v2_2, self.packindexfile_v2), + (self.packfile_v2_3_ascii, self.packindexfile_v2_3_ascii)): packfile, version, size = packinfo indexfile, version, size = indexinfo entity = PackEntity(packfile) @@ -193,22 +191,23 @@ def test_pack_entity(self, rw_dir): pack_path = tempfile.mktemp('', "pack", rw_dir) index_path = tempfile.mktemp('', 'index', rw_dir) iteration = 0 + def rewind_streams(): for obj in pack_objs: obj.stream.seek(0) - #END utility - for ppath, ipath, num_obj in zip((pack_path, )*2, (index_path, None), (len(pack_objs), None)): + # END utility + for ppath, ipath, num_obj in zip((pack_path, ) * 2, (index_path, None), (len(pack_objs), None)): pfile = open(ppath, 'wb') iwrite = None if ipath: ifile = open(ipath, 'wb') iwrite = ifile.write - #END handle ip + # END handle ip # make sure we rewind the streams ... we work on the same objects over and over again if iteration > 0: rewind_streams() - #END rewind streams + # END rewind streams iteration += 1 pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) @@ -230,8 +229,8 @@ def rewind_streams(): assert idx.packfile_checksum() == pack_sha assert idx.indexfile_checksum() == index_sha assert idx.size() == len(pack_objs) - #END verify files exist - #END for each packpath, indexpath pair + # END verify files exist + # END for each packpath, indexpath pair # verify the packs throughly rewind_streams() @@ -242,10 +241,9 @@ def rewind_streams(): for use_crc in range(2): assert entity.is_valid_stream(info.binsha, use_crc) # END for each crc mode - #END for each info + # END for each info assert count == len(pack_objs) - def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets # of course without really needing such a huge pack diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 44a557d53..96268252d 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -31,10 +31,12 @@ import os from io import BytesIO + class TestStream(TestBase): + """Test stream classes""" - data_sizes = (15, 10000, 1000*1024+512) + data_sizes = (15, 10000, 1000 * 1024 + 512) def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): """Make stream tests - the orig_stream is seekable, allowing it to be @@ -43,13 +45,13 @@ def _assert_stream_reader(self, stream, cdata, rewind_stream=lambda s: None): :param rewind_stream: function called to rewind the stream to make it ready for reuse""" ns = 10 - assert len(cdata) > ns-1, "Data must be larger than %i, was %i" % (ns, len(cdata)) + assert len(cdata) > ns - 1, "Data must be larger than %i, was %i" % (ns, len(cdata)) # read in small steps ss = len(cdata) // ns for i in range(ns): data = stream.read(ss) - chunk = cdata[i*ss:(i+1)*ss] + chunk = cdata[i * ss:(i + 1) * ss] assert data == chunk # END for each step rest = stream.read() @@ -136,7 +138,7 @@ def test_compressed_writer(self): self.failUnlessRaises(OSError, os.close, fd) # read everything back, compare to data we zip - fd = os.open(path, os.O_RDONLY|getattr(os, 'O_BINARY', 0)) + fd = os.open(path, os.O_RDONLY | getattr(os, 'O_BINARY', 0)) written_data = os.read(fd, os.path.getsize(path)) assert len(written_data) == os.path.getsize(path) os.close(fd) @@ -156,7 +158,7 @@ def test_decompress_reader_special_case(self): data = ostream.read() assert len(data) == ostream.size - # Putting it back in should yield nothing new - after all, we have + # Putting it back in should yield nothing new - after all, we have dump = mdb.store(IStream(ostream.type, ostream.size, BytesIO(data))) assert dump.hexsha == sha # end for each loose object sha to test diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index e79355aaf..1dee54461 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -16,6 +16,7 @@ class TestUtils(TestBase): + def test_basics(self): assert to_hex_sha(NULL_HEX_SHA) == NULL_HEX_SHA assert len(to_bin_sha(NULL_HEX_SHA)) == 20 @@ -73,7 +74,6 @@ def test_lockedfd(self): del(lfd) assert not os.path.isfile(lockfilepath) - # write data - concurrently lfd = LockedFD(my_file) olfd = LockedFD(my_file) diff --git a/gitdb/typ.py b/gitdb/typ.py index bc7ba5828..98d15f3ec 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -4,7 +4,7 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module containing information about types known to the database""" -str_blob_type = b'blob' +str_blob_type = b'blob' str_commit_type = b'commit' -str_tree_type = b'tree' -str_tag_type = b'tag' +str_tree_type = b'tree' +str_tag_type = b'tag' diff --git a/gitdb/util.py b/gitdb/util.py index 93ba7f0ec..5b451faad 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -11,10 +11,10 @@ from io import StringIO from smmap import ( - StaticWindowMapManager, - SlidingWindowMapManager, - SlidingWindowMapBuffer - ) + StaticWindowMapManager, + SlidingWindowMapManager, + SlidingWindowMapBuffer +) # initialize our global memory manager instance # Use it to free cached (and unused) resources. @@ -22,7 +22,7 @@ mman = StaticWindowMapManager() else: mman = SlidingWindowMapManager() -#END handle mman +# END handle mman import hashlib @@ -31,6 +31,7 @@ except ImportError: from struct import unpack, calcsize __calcsize_cache = dict() + def unpack_from(fmt, data, offset=0): try: size = __calcsize_cache[fmt] @@ -38,7 +39,7 @@ def unpack_from(fmt, data, offset=0): size = calcsize(fmt) __calcsize_cache[fmt] = size # END exception handling - return unpack(fmt, data[offset : offset + size]) + return unpack(fmt, data[offset: offset + size]) # END own unpack_from implementation @@ -67,8 +68,8 @@ def unpack_from(fmt, data, offset=0): fsync = os.fsync # Backwards compatibility imports -from gitdb.const import ( - NULL_BIN_SHA, +from gitdb.const import ( + NULL_BIN_SHA, NULL_HEX_SHA ) @@ -76,7 +77,9 @@ def unpack_from(fmt, data, offset=0): #{ compatibility stuff ... + class _RandomAccessStringIO(object): + """Wrapper to provide required functionality in case memory maps cannot or may not be used. This is only really required in python 2.4""" __slots__ = '_sio' @@ -96,6 +99,7 @@ def __getitem__(self, i): def __getslice__(self, start, end): return self.getvalue()[start:end] + def byte_ord(b): """ Return the integer representation of the byte string. This supports Python @@ -110,6 +114,7 @@ def byte_ord(b): #{ Routines + def make_sha(source=''.encode("ascii")): """A python2.4 workaround for the sha/hashlib module fiasco @@ -121,6 +126,7 @@ def make_sha(source=''.encode("ascii")): sha1 = sha.sha(source) return sha1 + def allocate_memory(size): """:return: a file-protocol accessible memory block of the given size""" if size == 0: @@ -134,7 +140,7 @@ def allocate_memory(size): # this of course may fail if the amount of memory is not available in # one chunk - would only be the case in python 2.4, being more likely on # 32 bit systems. - return _RandomAccessStringIO("\0"*size) + return _RandomAccessStringIO("\0" * size) # END handle memory allocation @@ -166,6 +172,7 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): return _RandomAccessStringIO(contents) return contents + def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): """Get the file contents at filepath as fast as possible @@ -178,25 +185,28 @@ def file_contents_ro_filepath(filepath, stream=False, allow_mmap=True, flags=0): **Note** for now we don't try to use O_NOATIME directly as the right value needs to be shared per database in fact. It only makes a real difference for loose object databases anyway, and they use it with the help of the ``flags`` parameter""" - fd = os.open(filepath, os.O_RDONLY|getattr(os, 'O_BINARY', 0)|flags) + fd = os.open(filepath, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags) try: return file_contents_ro(fd, stream, allow_mmap) finally: close(fd) # END assure file is closed + def sliding_ro_buffer(filepath, flags=0): """ :return: a buffer compatible object which uses our mapped memory manager internally ready to read the whole given filepath""" return SlidingWindowMapBuffer(mman.make_cursor(filepath), flags=flags) + def to_hex_sha(sha): """:return: hexified version of sha""" if len(sha) == 40: return sha return bin_to_hex(sha) + def to_bin_sha(sha): if len(sha) == 20: return sha @@ -209,6 +219,7 @@ def to_bin_sha(sha): #{ Utilities class LazyMixin(object): + """ Base class providing an interface to lazily retrieve attribute values upon first access. If slots are used, memory will only be reserved once the attribute @@ -240,6 +251,7 @@ def _set_cache_(self, attr): class LockedFD(object): + """ This class facilitates a safe read and write operation to a file on disk. If we write to 'file', we obtain a lock file at 'file.lock' and write to @@ -290,7 +302,7 @@ def open(self, write=False, stream=False): # try to open the lock file binary = getattr(os, 'O_BINARY', 0) - lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary + lockmode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary try: fd = os.open(self._lockfilepath(), lockmode, int("600", 8)) if not write: diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index a2640fd23..c08cab555 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -24,7 +24,7 @@ def buffer(obj, offset, size=None): return obj[offset:] else: # return memoryview(obj)[offset:offset+size] - return obj[offset:offset+size] + return obj[offset:offset + size] # end buffer reimplementation memoryview = memoryview diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 2d03ad30c..5855062bf 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -7,6 +7,7 @@ string_types = (basestring, ) text_type = unicode + def force_bytes(data, encoding="ascii"): if isinstance(data, bytes): return data @@ -16,6 +17,7 @@ def force_bytes(data, encoding="ascii"): return data + def force_text(data, encoding="utf-8"): if isinstance(data, text_type): return data diff --git a/setup.py b/setup.py index 4f8d1d5b2..6c67dc6aa 100755 --- a/setup.py +++ b/setup.py @@ -1,70 +1,74 @@ #!/usr/bin/env python -from distutils.core import setup, Extension +from distutils.core import setup, Extension from distutils.command.build_py import build_py from distutils.command.build_ext import build_ext -import os, sys +import os +import sys -# wow, this is a mixed bag ... I am pretty upset about all of this ... +# wow, this is a mixed bag ... I am pretty upset about all of this ... setuptools_build_py_module = None try: - # don't pull it in if we don't have to - if 'setuptools' in sys.modules: - import setuptools.command.build_py as setuptools_build_py_module - from setuptools.command.build_ext import build_ext + # don't pull it in if we don't have to + if 'setuptools' in sys.modules: + import setuptools.command.build_py as setuptools_build_py_module + from setuptools.command.build_ext import build_ext except ImportError: - pass + pass + class build_ext_nofail(build_ext): - """Doesn't fail when build our optional extensions""" - def run(self): - try: - build_ext.run(self) - except Exception: - print("Ignored failure when building extensions, pure python modules will be used instead") - # END ignore errors - + + """Doesn't fail when build our optional extensions""" + + def run(self): + try: + build_ext.run(self) + except Exception: + print("Ignored failure when building extensions, pure python modules will be used instead") + # END ignore errors + def get_data_files(self): - """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, - the line dealing with the ``plen`` has a bug which causes it to truncate too much. - It is fixed in the system interpreters as they receive patches, and shows how - bad it is if something doesn't have proper unittests. - The code here is a plain copy of the python2.6 version which works for all. - - Generate list of '(package,src_dir,build_dir,filenames)' tuples""" - data = [] - if not self.packages: - return data - - # this one is just for the setup tools ! They don't iniitlialize this variable - # when they should, but do it on demand using this method.Its crazy - if hasattr(self, 'analyze_manifest'): - self.analyze_manifest() - # END handle setuptools ... - - for package in self.packages: - # Locate package source directory - src_dir = self.get_package_dir(package) - - # Compute package build directory - build_dir = os.path.join(*([self.build_lib] + package.split('.'))) - - # Length of path to strip from found files - plen = 0 - if src_dir: - plen = len(src_dir)+1 - - # Strip directory from globbed filenames - filenames = [ - file[plen:] for file in self.find_data_files(package, src_dir) - ] - data.append((package, src_dir, build_dir, filenames)) - return data - + """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, + the line dealing with the ``plen`` has a bug which causes it to truncate too much. + It is fixed in the system interpreters as they receive patches, and shows how + bad it is if something doesn't have proper unittests. + The code here is a plain copy of the python2.6 version which works for all. + + Generate list of '(package,src_dir,build_dir,filenames)' tuples""" + data = [] + if not self.packages: + return data + + # this one is just for the setup tools ! They don't iniitlialize this variable + # when they should, but do it on demand using this method.Its crazy + if hasattr(self, 'analyze_manifest'): + self.analyze_manifest() + # END handle setuptools ... + + for package in self.packages: + # Locate package source directory + src_dir = self.get_package_dir(package) + + # Compute package build directory + build_dir = os.path.join(*([self.build_lib] + package.split('.'))) + + # Length of path to strip from found files + plen = 0 + if src_dir: + plen = len(src_dir) + 1 + + # Strip directory from globbed filenames + filenames = [ + file[plen:] for file in self.find_data_files(package, src_dir) + ] + data.append((package, src_dir, build_dir, filenames)) + return data + build_py.get_data_files = get_data_files if setuptools_build_py_module: - setuptools_build_py_module.build_py._get_data_files = get_data_files + setuptools_build_py_module.build_py._get_data_files = get_data_files # END apply setuptools patch too # NOTE: This is currently duplicated from the gitdb.__init__ module, as we cannot @@ -76,15 +80,15 @@ def get_data_files(self): version_info = (0, 6, 1) __version__ = '.'.join(str(i) for i in version_info) -setup(cmdclass={'build_ext':build_ext_nofail}, - name = "gitdb", - version = __version__, - description = "Git Object Database", - author = __author__, - author_email = __contact__, - url = __homepage__, - packages = ('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), - package_dir = {'gitdb':'gitdb'}, +setup(cmdclass={'build_ext': build_ext_nofail}, + name="gitdb", + version=__version__, + description="Git Object Database", + author=__author__, + author_email=__contact__, + url=__homepage__, + packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), + package_dir = {'gitdb': 'gitdb'}, ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, @@ -94,26 +98,26 @@ def get_data_files(self): # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ # Picked from - # http://pypi.python.org/pypi?:action=list_classifiers - #"Development Status :: 1 - Planning", - #"Development Status :: 2 - Pre-Alpha", - #"Development Status :: 3 - Alpha", - # "Development Status :: 4 - Beta", - "Development Status :: 5 - Production/Stable", - #"Development Status :: 6 - Mature", - #"Development Status :: 7 - Inactive", - "Environment :: Console", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Operating System :: POSIX", - "Operating System :: Microsoft :: Windows", - "Operating System :: MacOS :: MacOS X", - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - ],) + # http://pypi.python.org/pypi?:action=list_classifiers + #"Development Status :: 1 - Planning", + #"Development Status :: 2 - Pre-Alpha", + #"Development Status :: 3 - Alpha", + # "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", + #"Development Status :: 6 - Mature", + #"Development Status :: 7 - Inactive", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", +],) From c1998a074d2fd1773322e4595f30a5ecbbd54e32 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 15:23:00 +0100 Subject: [PATCH 0287/1392] Fixed python 3 performance regression It makes the difference between tests in 110s, or 11s --- doc/source/changes.rst | 5 +++++ smmap/__init__.py | 2 +- smmap/util.py | 5 +++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index d5ed8e378..6cf9c83fa 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,11 @@ Changelog ######### +********** +v0.8.4 +********** +- Fixed Python 3 performance regression + ********** v0.8.3 ********** diff --git a/smmap/__init__.py b/smmap/__init__.py index c494648d7..5f0e095cc 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 3) +version_info = (0, 8, 4) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience diff --git a/smmap/util.py b/smmap/util.py index e079a527a..6daa1fa12 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -23,9 +23,10 @@ except NameError: # Python 3 has no `buffer`; only `memoryview` def buffer(obj, offset, size): - # return memoryview(obj)[offset:offset+size] + # Actually, for gitpython this is fastest ... . + return memoryview(obj)[offset:offset+size] # doing it directly is much faster ! - return obj[offset:offset + size] + # return obj[offset:offset + size] def string_types(): From b1c9d3eb5b13f2feecb242701f5b4842184f6234 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 15:28:56 +0100 Subject: [PATCH 0288/1392] A minor fix after porting git-python over to PY3 It doesn't do anything (in terms of fixing an issue), but it should be more correct than what was there previously --- gitdb/utils/encoding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 5855062bf..4d270af9d 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -22,7 +22,7 @@ def force_text(data, encoding="utf-8"): if isinstance(data, text_type): return data - if isinstance(data, string_types): + if isinstance(data, bytes): return data.decode(encoding) if compat.PY3: From e0b0becd97afb9b7ac434c5fabdadd20070d643d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 15:55:09 +0100 Subject: [PATCH 0289/1392] Restore compatibility to python 3.0 to 3.4 --- doc/source/changes.rst | 5 +++++ smmap/__init__.py | 2 +- smmap/buf.py | 37 +++++++++++++++++++++++++++---------- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 6cf9c83fa..ec423694c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,11 @@ Changelog ######### +********** +v0.8.5 +********** +- Fixed Python 3.0-3.3 regression, which also causes smmap to become about 3 times slower depending on the code path. It's related to this bug (http://bugs.python.org/issue15958), which was fixed in python 3.4 + ********** v0.8.4 ********** diff --git a/smmap/__init__.py b/smmap/__init__.py index 5f0e095cc..46b0002e4 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 4) +version_info = (0, 8, 5) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience diff --git a/smmap/buf.py b/smmap/buf.py index 17d2d369b..b3b71c49c 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -3,6 +3,8 @@ __all__ = ["SlidingWindowMapBuffer"] +import sys + try: bytes except NameError: @@ -79,16 +81,31 @@ def __getslice__(self, i, j): ofs = i # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower # in the previous iteration of this code - md = list() - while l: - c.use_region(ofs, l) - assert c.is_valid() - d = c.buffer()[:l] - ofs += len(d) - l -= len(d) - md.append(d) - # END while there are bytes to read - return bytes().join(md) + pyvers = sys.version_info[:2] + if (3, 0) <= pyvers <= (3, 3): + # Memory view cannot be joined below python 3.4 ... + out = bytes() + while l: + c.use_region(ofs, l) + assert c.is_valid() + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + # This is slower than the join ... but what can we do ... + out += d + # END while there are bytes to read + return out + else: + md = list() + while l: + c.use_region(ofs, l) + assert c.is_valid() + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + md.append(d) + # END while there are bytes to read + return bytes().join(md) # END fast or slow path #{ Interface From fdc1d68b01f0d5dd601cdcc29df0eee19787d7c9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 17:02:35 +0100 Subject: [PATCH 0290/1392] Fixed python 3 compatibility issue that only showed on windows And bumped version to 0.6.2 --- gitdb/__init__.py | 2 +- gitdb/util.py | 12 ++++++------ setup.py | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 791a2ef2f..020a5795f 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 1) +version_info = (0, 6, 2) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/util.py b/gitdb/util.py index 5b451faad..8f80156b2 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -8,7 +8,7 @@ import sys import errno -from io import StringIO +from io import BytesIO from smmap import ( StaticWindowMapManager, @@ -78,14 +78,14 @@ def unpack_from(fmt, data, offset=0): #{ compatibility stuff ... -class _RandomAccessStringIO(object): +class _RandomAccessBytesIO(object): """Wrapper to provide required functionality in case memory maps cannot or may not be used. This is only really required in python 2.4""" __slots__ = '_sio' def __init__(self, buf=''): - self._sio = StringIO(buf) + self._sio = BytesIO(buf) def __getattr__(self, attr): return getattr(self._sio, attr) @@ -130,7 +130,7 @@ def make_sha(source=''.encode("ascii")): def allocate_memory(size): """:return: a file-protocol accessible memory block of the given size""" if size == 0: - return _RandomAccessStringIO('') + return _RandomAccessBytesIO(b'') # END handle empty chunks gracefully try: @@ -140,7 +140,7 @@ def allocate_memory(size): # this of course may fail if the amount of memory is not available in # one chunk - would only be the case in python 2.4, being more likely on # 32 bit systems. - return _RandomAccessStringIO("\0" * size) + return _RandomAccessBytesIO(b"\0" * size) # END handle memory allocation @@ -169,7 +169,7 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): # read manully contents = os.read(fd, os.fstat(fd).st_size) if stream: - return _RandomAccessStringIO(contents) + return _RandomAccessBytesIO(contents) return contents diff --git a/setup.py b/setup.py index 6c67dc6aa..12c936a17 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 1) +version_info = (0, 6, 2) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext': build_ext_nofail}, @@ -92,8 +92,8 @@ def get_data_files(self): ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], license = "BSD License", zip_safe=False, - requires=('smmap (>=0.8.3)', ), - install_requires=('smmap >= 0.8.3'), + requires=('smmap (>=0.8.5)', ), + install_requires=('smmap >= 0.8.5'), long_description = """GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ From cb72f81e1407a86d85215a7fba4c2905c2451e0c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 18:17:23 +0100 Subject: [PATCH 0291/1392] Added coverage configuration Adjusted sublime project too --- .coveragerc | 11 +++++++++++ .gitignore | 3 ++- .travis.yml | 2 +- etc/sublime-text/gitdb.sublime-project | 24 +++++++----------------- 4 files changed, 21 insertions(+), 19 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..45d72abc9 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,11 @@ +[run] +source = gitdb + +; to make nosetests happy +[report] +omit = + */smmap/* + */yaml* + */tests/* + */python?.?/* + */site-packages/nose/* \ No newline at end of file diff --git a/.gitignore b/.gitignore index c6247dbb0..e0b4e8579 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ MANIFEST +.coverage build/ dist/ *.pyc *.o *.so .noseids -*.sublime-workspace \ No newline at end of file +*.sublime-workspace diff --git a/.travis.yml b/.travis.yml index 761edc19b..d436229f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ git: install: - pip install coveralls script: - - nosetests -v + - nosetests -v --with-coverage after_success: - coveralls diff --git a/etc/sublime-text/gitdb.sublime-project b/etc/sublime-text/gitdb.sublime-project index bc0e37f0a..d0e2e5132 100644 --- a/etc/sublime-text/gitdb.sublime-project +++ b/etc/sublime-text/gitdb.sublime-project @@ -15,7 +15,10 @@ "folder_exclude_patterns" : [ ".git", "cover", - "gitdb/ext" + "gitdb/ext", + "dist", + "doc/build", + ".tox" ] }, // SMMAP @@ -32,22 +35,9 @@ "folder_exclude_patterns" : [ ".git", "cover", - ] - }, - // ASYNC - //////// - { - "follow_symlinks": true, - "path": "../../gitdb/ext/async", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", + "dist", + "doc/build", + ".tox" ] }, ] From de96c522ff20fa99d13128784a393b619dd0b33b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 18:26:18 +0100 Subject: [PATCH 0292/1392] Fixed yet another issue with smmap's latest changes Now we deal with memory views as well ... --- gitdb/pack.py | 9 +++++++-- gitdb/utils/compat.py | 8 ++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index b4ba7876c..d2666d601 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -62,7 +62,12 @@ from binascii import crc32 from gitdb.const import NULL_BYTE -from gitdb.utils.compat import izip, buffer, xrange +from gitdb.utils.compat import ( + izip, + buffer, + xrange, + to_bytes +) import tempfile import array @@ -864,7 +869,7 @@ def collect_streams_at_offset(self, offset): stream = streams[-1] while stream.type_id in delta_types: if stream.type_id == REF_DELTA: - sindex = self._index.sha_to_index(stream.delta_info) + sindex = self._index.sha_to_index(to_bytes(stream.delta_info)) if sindex is None: break stream = self._pack.stream(self._index.offset(sindex)) diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index c08cab555..a7899cb14 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -15,6 +15,9 @@ # Python 2 buffer = buffer memoryview = buffer + # Assume no memory view ... + def to_bytes(i): + return i except NameError: # Python 3 has no `buffer`; only `memoryview` # However, it's faster to just slice the object directly, maybe it keeps a view internally @@ -26,6 +29,11 @@ def buffer(obj, offset, size=None): # return memoryview(obj)[offset:offset+size] return obj[offset:offset + size] # end buffer reimplementation + # smmap can return memory view objects, which can't be compared as buffers/bytes can ... + def to_bytes(i): + if isinstance(i, memoryview): + return i.tobytes() + return i memoryview = memoryview From a64e64a9735e8067ca196ab19711d136bd9b309c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 18:27:22 +0100 Subject: [PATCH 0293/1392] Bumped version to 0.6.3 --- gitdb/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 020a5795f..2ba8725eb 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 2) +version_info = (0, 6, 3) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 12c936a17..e634e37f2 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 2) +version_info = (0, 6, 3) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext': build_ext_nofail}, From 9b3a34b7d00285cf9028d19e82de6b155d0096c7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 6 Jan 2015 18:53:52 +0100 Subject: [PATCH 0294/1392] Improved coverage configuration --- .coveragerc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.coveragerc b/.coveragerc index 45d72abc9..71b6ef701 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,9 +3,5 @@ source = gitdb ; to make nosetests happy [report] -omit = - */smmap/* - */yaml* - */tests/* - */python?.?/* - */site-packages/nose/* \ No newline at end of file +include = */gitdb/* +omit = */gitdb/ext/* From 70fae1f98bb7d44b58d94a183e9eb8b590bc23bf Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jan 2015 16:01:06 +0100 Subject: [PATCH 0295/1392] Initial attempt to fix resource usage Reference counting is now done manually, but it seems that things can still go wrong at least during testing --- doc/source/changes.rst | 6 ++++++ smmap/mman.py | 27 +++++++++++++-------------- smmap/test/test_mman.py | 26 +++++++++++++------------- smmap/test/test_util.py | 9 +-------- smmap/util.py | 33 +++++++++++++++++++-------------- 5 files changed, 52 insertions(+), 49 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index ec423694c..f9f328757 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +********** +v0.8.6 +********** +- Fixed issue with resources never being freed as mmaps were never closed. +- Client counting is now done manually, instead of relying on pyton's reference count + ********** v0.8.5 ********** diff --git a/smmap/mman.py b/smmap/mman.py index c7a459558..6e8b9eeaf 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -8,7 +8,6 @@ buffer, ) -from weakref import ref import sys from functools import reduce @@ -55,8 +54,7 @@ def _destroy(self): # Actual client count, which doesn't include the reference kept by the manager, nor ours # as we are about to be deleted try: - num_clients = self._rlist.client_count() - 2 - if num_clients == 0 and len(self._rlist) == 0: + if len(self._rlist) == 0: # Free all resources associated with the mapped file self._manager._fdict.pop(self._rlist.path_or_fd()) # END remove regions list from manager @@ -78,7 +76,7 @@ def _copy_from(self, rhs): self._size = rhs._size if self._region is not None: - self._region.increment_usage_count() + self._region.increment_client_count() # END handle regions def __copy__(self): @@ -126,20 +124,22 @@ def use_region(self, offset=0, size=0, flags=0): if need_region: self._region = man._obtain_region(self._rlist, offset, size, flags, False) + self._region.increment_client_count() # END need region handling - self._region.increment_usage_count() self._ofs = offset - self._region._b self._size = min(size, self._region.ofs_end() - offset) return self def unuse_region(self): - """Unuse the ucrrent region. Does nothing if we have no current region + """Unuse the current region. Does nothing if we have no current region **Note:** the cursor unuses the region automatically upon destruction. It is recommended to un-use the region once you are done reading from it in persistent cursors as it helps to free up resource more quickly""" + if self._region is not None: + self._region.increment_client_count(-1) self._region = None # note: should reset ofs and size, but we spare that for performance. Its not # allowed to query information if we are not valid ! @@ -184,12 +184,10 @@ def size(self): """:return: amount of bytes we point to""" return self._size - def region_ref(self): - """:return: weak ref to our mapped region. + def region(self): + """:return: our mapped region, or None if nothing is mapped yet :raise AssertionError: if we have no current region. This is only useful for debugging""" - if self._region is None: - raise AssertionError("region not set") - return ref(self._region) + return self._region def includes_ofs(self, ofs): """:return: True if the given absolute offset is contained in the cursors @@ -311,8 +309,8 @@ def _collect_lru_region(self, size): lru_list = None for regions in self._fdict.values(): for region in regions: - # check client count - consider that we keep one reference ourselves ! - if (region.client_count() - 2 == 0 and + # check client count - if it's 1, it's just us + if (region.client_count() == 1 and (lru_region is None or region._uc < lru_region._uc)): lru_region = region lru_list = regions @@ -326,6 +324,7 @@ def _collect_lru_region(self, size): num_found += 1 del(lru_list[lru_list.index(lru_region)]) + lru_region.increment_client_count(-1) self._memory_size -= lru_region.size() self._handle_count -= 1 # END while there is more memory to free @@ -449,7 +448,7 @@ def force_map_handle_removal_win(self, base_path): for path, rlist in self._fdict.items(): if path.startswith(base_path): for region in rlist: - region._mf.close() + region.release() num_closed += 1 # END path matches # END for each path diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index b718b065f..c8b9c703e 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -119,21 +119,21 @@ def test_memman_operation(self): # window size is 0 for static managers, hence size will be 0. We take that into consideration size = man.window_size() // 2 assert c.use_region(base_offset, size).is_valid() - rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us + rr = c.region() + assert rr.client_count() == 2 # the manager and the cursor and us assert man.num_open_files() == 1 assert man.num_file_handles() == 1 - assert man.mapped_memory_size() == rr().size() + assert man.mapped_memory_size() == rr.size() # assert c.size() == size # the cursor may overallocate in its static version assert c.ofs_begin() == base_offset - assert rr().ofs_begin() == 0 # it was aligned and expanded + assert rr.ofs_begin() == 0 # it was aligned and expanded if man.window_size(): # but isn't larger than the max window (aligned) - assert rr().size() == align_to_mmap(man.window_size(), True) + assert rr.size() == align_to_mmap(man.window_size(), True) else: - assert rr().size() == fc.size + assert rr.size() == fc.size # END ignore static managers which dont use windows and are aligned to file boundaries assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] @@ -141,7 +141,7 @@ def test_memman_operation(self): # obtain second window, which spans the first part of the file - it is a still the same window nsize = (size or fc.size) - 10 assert c.use_region(0, nsize).is_valid() - assert c.region_ref()() == rr() + assert c.region() == rr assert man.num_file_handles() == 1 assert c.size() == nsize assert c.ofs_begin() == 0 @@ -154,15 +154,15 @@ def test_memman_operation(self): if man.window_size(): assert man.num_file_handles() == 2 assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager + assert c.region() is not rr # old region is still available, but has not curser ref anymore + assert rr.client_count() == 1 # only held by manager else: assert c.size() < fc.size # END ignore static managers which only have one handle per file - rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file + rr = c.region() + assert rr.client_count() == 2 # manager + cursor + assert rr.ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr.ofs_end() <= fc.size # it cannot be larger than the file assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] # unising a region makes the cursor invalid diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 0bbf91b65..0a162607e 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -88,12 +88,7 @@ def test_region(self): # auto-refcount assert rfull.client_count() == 1 rfull2 = rfull - assert rfull.client_count() == 2 - - # usage - assert rfull.usage_count() == 0 - rfull.increment_usage_count() - assert rfull.usage_count() == 1 + assert rfull.client_count() == 1, "no auto-counting" # window constructor w = MapWindow.from_region(rfull) @@ -106,8 +101,6 @@ def test_region_list(self): for item in (fc.path, fd): ml = MapRegionList(item) - assert ml.client_count() == 1 - assert len(ml) == 0 assert ml.path_or_fd() == item assert ml.file_size() == fc.size diff --git a/smmap/util.py b/smmap/util.py index 6daa1fa12..1d20e5040 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -177,6 +177,8 @@ def __init__(self, path_or_fd, ofs, size, flags=0): os.close(fd) # END only close it if we opened it # END close file handle + # We assume the first one to use us keeps us around + self.increment_client_count() def _read_into_memory(self, fd, offset, size): """:return: string data as read from the given file descriptor, offset and size """ @@ -222,17 +224,25 @@ def includes_ofs(self, ofs): def client_count(self): """:return: number of clients currently using this region""" - from sys import getrefcount - # -1: self on stack, -1 self in this method, -1 self in getrefcount - return getrefcount(self) - 3 - - def usage_count(self): - """:return: amount of usages so far""" return self._uc - def increment_usage_count(self): - """Adjust the usage count by the given positive or negative offset""" - self._uc += 1 + def increment_client_count(self, ofs = 1): + """Adjust the usage count by the given positive or negative offset. + If usage count equals 0, we will auto-release our resources + :return: True if we released resources, False otherwise. In the latter case, we can still be used""" + self._uc += ofs + assert self._uc > -1, "Increments must match decrements, usage counter negative: %i" % self._uc + + if self.client_count() == 0: + self.release() + return True + else: + return False + # end handle release + + def release(self): + """Release all resources this instance might hold. Must only be called if there usage_count() is zero""" + self._mf.close() # re-define all methods which need offset adjustments in compatibility mode if _need_compat_layer: @@ -268,11 +278,6 @@ def __init__(self, path_or_fd): self._path_or_fd = path_or_fd self._file_size = None - def client_count(self): - """:return: amount of clients which hold a reference to this instance""" - from sys import getrefcount - return getrefcount(self) - 3 - def path_or_fd(self): """:return: path or file descriptor we are attached to""" return self._path_or_fd From f071ffdcacbafff648cd29d6f75fe27c47f53210 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jan 2015 16:57:50 +0100 Subject: [PATCH 0296/1392] All tests work, bumped version --- doc/source/changes.rst | 2 +- smmap/__init__.py | 2 +- smmap/buf.py | 5 +++++ smmap/mman.py | 7 +++++-- smmap/test/test_buf.py | 3 ++- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index f9f328757..f99e85fb7 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -3,7 +3,7 @@ Changelog ######### ********** -v0.8.6 +v0.9.0 ********** - Fixed issue with resources never being freed as mmaps were never closed. - Client counting is now done manually, instead of relying on pyton's reference count diff --git a/smmap/__init__.py b/smmap/__init__.py index 46b0002e4..e711bbbf3 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 8, 5) +version_info = (0, 9, 0) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience diff --git a/smmap/buf.py b/smmap/buf.py index b3b71c49c..e6f24341d 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -93,6 +93,7 @@ def __getslice__(self, i, j): l -= len(d) # This is slower than the join ... but what can we do ... out += d + del(d) # END while there are bytes to read return out else: @@ -103,6 +104,10 @@ def __getslice__(self, i, j): d = c.buffer()[:l] ofs += len(d) l -= len(d) + # Make sure we don't keep references, as c.use_region() might attempt to free resources, but + # can't unless we use pure bytes + if hasattr(d, 'tobytes'): + d = d.tobytes() md.append(d) # END while there are bytes to read return bytes().join(md) diff --git a/smmap/mman.py b/smmap/mman.py index 6e8b9eeaf..7180c75b3 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -58,7 +58,7 @@ def _destroy(self): # Free all resources associated with the mapped file self._manager._fdict.pop(self._rlist.path_or_fd()) # END remove regions list from manager - except TypeError: + except (TypeError, KeyError): # sometimes, during shutdown, getrefcount is None. Its possible # to re-import it, however, its probably better to just ignore # this python problem (for now). @@ -70,11 +70,14 @@ def _destroy(self): def _copy_from(self, rhs): """Copy all data from rhs into this instance, handles usage count""" self._manager = rhs._manager - self._rlist = rhs._rlist + self._rlist = type(rhs._rlist)(rhs._rlist) self._region = rhs._region self._ofs = rhs._ofs self._size = rhs._size + for region in self._rlist: + region.increment_client_count() + if self._region is not None: self._region.increment_client_count() # END handle regions diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 03377154b..984b43254 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -12,7 +12,6 @@ from time import time import sys import os -import logging man_optimal = SlidingWindowMapManager() @@ -104,6 +103,7 @@ def test_basics(self): assert len(d) == ofs_end - ofs_start assert d == data[ofs_start:ofs_end] num_bytes += len(d) + del d else: pos = randint(0, fsize) assert buf[pos] == data[pos] @@ -122,6 +122,7 @@ def test_basics(self): % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), file=sys.stderr) # END handle access mode + del buf # END for each manager # END for each input os.close(fd) From a38efa84daef914e4de58d1905a500d8d14aaf45 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jan 2015 18:03:33 +0100 Subject: [PATCH 0297/1392] Artificially restrict test-runs to assure we don't leak handles --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index cb0c16e44..b967f502e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,8 @@ python: install: - pip install coveralls script: + - ulimit -n 48 + - ulimit -n - nosetests --with-coverage after_success: - coveralls From 560a211001064261eb25ca874980591790fb7986 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jan 2015 18:09:45 +0100 Subject: [PATCH 0298/1392] Fixed possible file-handle leak Configured travis to artificially restrict handle count to protect from regression in that regard --- .travis.yml | 2 ++ gitdb/stream.py | 14 +++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d436229f0..8cab8225f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,8 @@ git: install: - pip install coveralls script: + - ulimit -n 48 + - ulimit -n - nosetests -v --with-coverage after_success: - coveralls diff --git a/gitdb/stream.py b/gitdb/stream.py index 4478a0f44..d855257cd 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -91,9 +91,7 @@ def _set_cache_(self, attr): self._parse_header_info() def __del__(self): - if self._close: - self._m.close() - # END handle resource freeing + self.close() def _parse_header_info(self): """If this stream contains object data, parse the header info and skip the @@ -141,6 +139,16 @@ def data(self): """:return: random access compatible data we are working on""" return self._m + def close(self): + """Close our underlying stream of compressed bytes if this was allowed during initialization + :return: True if we closed the underlying stream + :note: can be called safely + """ + if self._close: + self._m.close() + self._close = False + # END handle resource freeing + def compressed_bytes_read(self): """ :return: number of compressed bytes read. This includes the bytes it From be294278a0087f21d565a1084fb220ff936ae0bd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 7 Jan 2015 19:57:19 +0100 Subject: [PATCH 0299/1392] Protected stream closure against possibilty of being a bytes For some reason, it gets bytes where it did expect a stream ... . Probably I should have figured out where this was input, instead of fixing it the brutal way --- gitdb/db/loose.py | 3 ++- gitdb/stream.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index e924080ef..4732b56da 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -159,7 +159,8 @@ def info(self, sha): typ, size = loose_object_header_info(m) return OInfo(sha, typ, size) finally: - m.close() + if hasattr(m, 'close'): + m.close() # END assure release of system resources def stream(self, sha): diff --git a/gitdb/stream.py b/gitdb/stream.py index d855257cd..826def387 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -145,7 +145,8 @@ def close(self): :note: can be called safely """ if self._close: - self._m.close() + if hasattr(self._m, 'close'): + self._m.close() self._close = False # END handle resource freeing From 7bde7b098b07291227fcbc4eb900ebf13c9191a2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 8 Jan 2015 17:34:53 +0100 Subject: [PATCH 0300/1392] Fixed up tests to use the GITDB_TEST_GIT_REPO_BASE at all times I have verified that all tests are working, even without a parent git repository, as long as the said environment variable is set. Fixes #16 --- gitdb/exc.py | 12 +++++++----- gitdb/test/db/test_git.py | 3 ++- gitdb/test/db/test_ref.py | 2 +- gitdb/test/lib.py | 29 ++++++++++++++++++++++++++++- gitdb/test/performance/lib.py | 30 ++---------------------------- gitdb/test/test_example.py | 8 +++----- 6 files changed, 43 insertions(+), 41 deletions(-) diff --git a/gitdb/exc.py b/gitdb/exc.py index d58442f2b..817ac7b62 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -12,12 +12,10 @@ class ODBError(Exception): class InvalidDBRoot(ODBError): - """Thrown if an object database cannot be initialized at the given path""" class BadObject(ODBError): - """The object with the given SHA does not exist. Instantiate with the failed sha""" @@ -25,19 +23,23 @@ def __str__(self): return "BadObject: %s" % to_hex_sha(self.args[0]) -class ParseError(ODBError): +class BadName(ODBError): + """A name provided to rev_parse wasn't understood""" + + def __str__(self): + return "Ref '%s' did not resolve to an object" % self.args[0] + +class ParseError(ODBError): """Thrown if the parsing of a file failed due to an invalid format""" class AmbiguousObjectName(ODBError): - """Thrown if a possibly shortened name does not uniquely represent a single object in the database""" class BadObjectType(ODBError): - """The object had an unsupported type""" diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index f96206744..f28ffb761 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -2,6 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php +import os from gitdb.test.db.lib import ( TestDBBase, fixture_path, @@ -16,7 +17,7 @@ class TestGitDB(TestDBBase): def test_reading(self): - gdb = GitDB(fixture_path('../../../.git/objects')) + gdb = GitDB(os.path.join(self.gitrepopath, 'objects')) # we have packs and loose objects, alternates doesn't necessarily exist assert 1 < len(gdb.databases()) < 4 diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index b774bafe4..25cf37d1b 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -40,7 +40,7 @@ def test_writing(self, path): # setup alternate file # add two, one is invalid - own_repo_path = fixture_path('../../../.git/objects') # use own repo + own_repo_path = os.path.join(self.gitrepopath, 'objects') # use own repo self.make_alt_file(alt_path, [own_repo_path, "invalid/path"]) rdb.update_cache() assert len(rdb.databases()) == 1 diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index c4acd9218..a089eac6f 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -18,14 +18,41 @@ import shutil import os import gc +import logging from functools import wraps #{ Bases class TestBase(unittest.TestCase): + """Base class for all tests - """Base class for all tests""" + TestCase providing access to readonly repositories using the following member variables. + + * gitrepopath + + * read-only base path of the git source repository, i.e. .../git/.git + """ + + #{ Invvariants + k_env_git_repo = "GITDB_TEST_GIT_REPO_BASE" + #} END invariants + + @classmethod + def setUpClass(cls): + try: + super(TestBase, cls).setUpClass() + except AttributeError: + pass + + cls.gitrepopath = os.environ.get(cls.k_env_git_repo) + if not cls.gitrepopath: + logging.info( + "You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository", cls.k_env_git_repo) + ospd = os.path.dirname + cls.gitrepopath = os.path.join(ospd(ospd(ospd(__file__))), '.git') + # end assure gitrepo is set + assert cls.gitrepopath.endswith('.git') #} END bases diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index cbc52bc77..fa4dd209d 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -3,41 +3,15 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Contains library functions""" -import os -import logging from gitdb.test.lib import TestBase -#{ Invvariants -k_env_git_repo = "GITDB_TEST_GIT_REPO_BASE" -#} END invariants - #{ Base Classes class TestBigRepoR(TestBase): - - """TestCase providing access to readonly 'big' repositories using the following - member variables: - - * gitrepopath - - * read-only base path of the git source repository, i.e. .../git/.git""" - - def setUp(self): - try: - super(TestBigRepoR, self).setUp() - except AttributeError: - pass - - self.gitrepopath = os.environ.get(k_env_git_repo) - if not self.gitrepopath: - logging.info( - "You can set the %s environment variable to a .git repository of your choice - defaulting to the gitdb repository", k_env_git_repo) - ospd = os.path.dirname - self.gitrepopath = os.path.join(ospd(ospd(ospd(ospd(__file__)))), '.git') - # end assure gitrepo is set - assert self.gitrepopath.endswith('.git') + """A placeholder in case we want to add additional functionality to all performance test-cases + """ #} END base classes diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index ed0a885bf..6e80bf5c6 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -3,10 +3,8 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Module with examples from the tutorial section of the docs""" -from gitdb.test.lib import ( - TestBase, - fixture_path -) +import os +from gitdb.test.lib import TestBase from gitdb import IStream from gitdb.db import LooseObjectDB @@ -16,7 +14,7 @@ class TestExamples(TestBase): def test_base(self): - ldb = LooseObjectDB(fixture_path("../../../.git/objects")) + ldb = LooseObjectDB(os.path.join(self.gitrepopath, 'objects')) for sha1 in ldb.sha_iter(): oinfo = ldb.info(sha1) From f2233fbf40f3f69309ce5cc714e99fcbdcd33ec3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 8 Jan 2015 17:49:05 +0100 Subject: [PATCH 0301/1392] Removed unused imports - should have been in the last commit obviously --- gitdb/test/db/test_git.py | 1 - gitdb/test/db/test_ref.py | 1 - 2 files changed, 2 deletions(-) diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index f28ffb761..2bda18f84 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -5,7 +5,6 @@ import os from gitdb.test.db.lib import ( TestDBBase, - fixture_path, with_rw_directory ) from gitdb.exc import BadObject diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 25cf37d1b..0e90f938b 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -5,7 +5,6 @@ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory, - fixture_path ) from gitdb.db import ReferenceDB From a88a777df3909a61be97f1a7b1194dad6de25702 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 8 Jan 2015 18:25:46 +0100 Subject: [PATCH 0302/1392] Make tests independent of actual repository data Therefore, hardcoded sha's are not allowed anymore, as the contents of the repository is unknown. Fixes #16, for real this time ;) --- gitdb/test/db/test_git.py | 7 ++++--- gitdb/test/db/test_ref.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 2bda18f84..acc0f153f 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -10,7 +10,7 @@ from gitdb.exc import BadObject from gitdb.db import GitDB from gitdb.base import OStream, OInfo -from gitdb.util import hex_to_bin, bin_to_hex +from gitdb.util import bin_to_hex class TestGitDB(TestDBBase): @@ -22,7 +22,7 @@ def test_reading(self): assert 1 < len(gdb.databases()) < 4 # access should be possible - gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") + gitdb_sha = next(gdb.sha_iter()) assert isinstance(gdb.info(gitdb_sha), OInfo) assert isinstance(gdb.stream(gitdb_sha), OStream) ni = 50 @@ -35,7 +35,8 @@ def test_reading(self): # have a separate test module # test partial shas # this one as uneven and quite short - assert gdb.partial_to_complete_sha_hex('155b6') == hex_to_bin("155b62a9af0aa7677078331e111d0f7aa6eb4afc") + gitdb_sha_hex = bin_to_hex(gitdb_sha) + assert gdb.partial_to_complete_sha_hex(gitdb_sha_hex[:5]) == gitdb_sha # mix even/uneven hexshas for i, binsha in enumerate(sha_list): diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 0e90f938b..6bac245c1 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -45,7 +45,7 @@ def test_writing(self, path): assert len(rdb.databases()) == 1 # we should now find a default revision of ours - gitdb_sha = hex_to_bin("5690fd0d3304f378754b23b098bd7cb5f4aa1976") + gitdb_sha = next(rdb.sha_iter()) assert rdb.has_object(gitdb_sha) # remove valid From 13ad9b1199331a35e23f65c735acf482be09eae3 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 8 Jan 2015 13:10:28 -0500 Subject: [PATCH 0303/1392] minor spell fixes + empty line unification + comparison for python 2.6 --- gitdb/base.py | 6 +++--- gitdb/exc.py | 2 -- gitdb/stream.py | 6 +++--- gitdb/util.py | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/gitdb/base.py b/gitdb/base.py index 5760b8af0..42e71d0fa 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -19,7 +19,7 @@ class OInfo(tuple): - """Carries information about an object in an ODB, provding information + """Carries information about an object in an ODB, providing information about the binary sha of the object, the type_string as well as the uncompressed size in bytes. @@ -29,7 +29,7 @@ class OInfo(tuple): assert dbi[1] == dbi.type assert dbi[2] == dbi.size - The type is designed to be as lighteight as possible.""" + The type is designed to be as lightweight as possible.""" __slots__ = tuple() def __new__(cls, sha, type, size): @@ -69,7 +69,7 @@ class OPackInfo(tuple): does not include a sha. Additionally, the pack_offset is the absolute offset into the packfile at which - all object information is located. The data_offset property points to the abosolute + all object information is located. The data_offset property points to the absolute location in the pack at which that actual data stream can be found.""" __slots__ = tuple() diff --git a/gitdb/exc.py b/gitdb/exc.py index 817ac7b62..947e5d8bf 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -7,7 +7,6 @@ class ODBError(Exception): - """All errors thrown by the object database""" @@ -44,5 +43,4 @@ class BadObjectType(ODBError): class UnsupportedOperation(ODBError): - """Thrown if the given operation cannot be supported by the object database""" diff --git a/gitdb/stream.py b/gitdb/stream.py index 826def387..aaf5820bf 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -62,7 +62,7 @@ class DecompressMemMapReader(LazyMixin): hence we try to find a good tradeoff between allocation time and number of times we actually allocate. An own zlib implementation would be good here to better support streamed reading - it would only need to keep the mmap - and decompress it into chunks, thats all ... """ + and decompress it into chunks, that's all ... """ __slots__ = ('_m', '_zip', '_buf', '_buflen', '_br', '_cws', '_cwe', '_s', '_close', '_cbr', '_phi') @@ -128,7 +128,7 @@ def new(self, m, close_on_deletion=False): This method parses the object header from m and returns the parsed type and size, as well as the created stream instance. - :param m: memory map on which to oparate. It must be object data ( header + contents ) + :param m: memory map on which to operate. It must be object data ( header + contents ) :param close_on_deletion: if True, the memory map will be closed once we are being deleted""" inst = DecompressMemMapReader(m, close_on_deletion, 0) @@ -175,7 +175,7 @@ def compressed_bytes_read(self): # Only scrub the stream forward if we are officially done with the # bytes we were to have. if self._br == self._s and not self._zip.unused_data: - # manipulate the bytes-read to allow our own read method to coninute + # manipulate the bytes-read to allow our own read method to continue # but keep the window at its current position self._br = 0 if hasattr(self._zip, 'status'): diff --git a/gitdb/util.py b/gitdb/util.py index 8f80156b2..242be4405 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -18,7 +18,7 @@ # initialize our global memory manager instance # Use it to free cached (and unused) resources. -if sys.version_info[1] < 6: +if sys.version_info < (2, 6): mman = StaticWindowMapManager() else: mman = SlidingWindowMapManager() From 5b0dc5f89a666f450f39ef0002cf6d1761ecfca8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 12 Jan 2015 19:10:42 +0100 Subject: [PATCH 0304/1392] Adjusted stream logic to make it work on all tested platforms ... . As taken from https://github.com/gitpython-developers/gitdb/blob/master/gitdb/stream.py#L292 -> NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. They are thorough, and I assume it is truly working. Why is this logic as convoluted as it is ? Please look at the table in https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. Bascially, on py2.6, you want to use branch 1, whereas on all other python version, the second branch will be the one that works. However, the zlib VERSIONs as well as the platform check is used to further match the entries in the table in the github issue. This is it ... it was the only way I could make this work everywhere. IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . <- Fixes #19 --- gitdb/fun.py | 2 +- gitdb/pack.py | 1 + gitdb/stream.py | 12 +++++++++--- setup.py | 1 + 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index 17da4e5da..ac9d99395 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -426,7 +426,7 @@ def pack_object_header_info(data): s = 4 # starting bit-shift size if PY3: while c & 0x80: - c = data[i] + c = byte_ord(data[i]) i += 1 size += (c & 0x7f) << s s += 7 diff --git a/gitdb/pack.py b/gitdb/pack.py index d2666d601..511e5571c 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -552,6 +552,7 @@ def _iter_objects(self, start_offset, as_stream=True): # the amount of compressed bytes we need to get to the next offset stream_copy(ostream.read, null.write, ostream.size, chunk_size) + assert ostream.stream._br == ostream.size cur_offset += (data_offset - ostream.pack_offset) + ostream.stream.compressed_bytes_read() # if a stream is requested, reset it beforehand diff --git a/gitdb/stream.py b/gitdb/stream.py index aaf5820bf..04dd79f90 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -289,11 +289,18 @@ def read(self, size=-1): # if we hit the end of the stream # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. # They are thorough, and I assume it is truly working. - if PY26: + # Why is this logic as convoluted as it is ? Please look at the table in + # https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. + # Bascially, on py2.6, you want to use branch 1, whereas on all other python version, the second branch + # will be the one that works. + # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the + # table in the github issue. This is it ... it was the only way I could make this work everywhere. + # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . + if PY26 or ((zlib.ZLIB_VERSION == '1.2.7' or zlib.ZLIB_VERSION == '1.2.5') and not sys.platform == 'darwin'): unused_datalen = len(self._zip.unconsumed_tail) else: unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) - # end handle very special case ... + # # end handle very special case ... self._cbr += len(indata) - unused_datalen self._br += len(dcompdat) @@ -374,7 +381,6 @@ def _set_cache_too_slow_without_c(self, attr): # Aggregate all deltas into one delta in reverse order. Hence we take # the last delta, and reverse-merge its ancestor delta, until we receive # the final delta data stream. - # print "Handling %i delta streams, sizes: %s" % (len(self._dstreams), [ds.size for ds in self._dstreams]) dcl = connect_deltas(self._dstreams) # call len directly, as the (optional) c version doesn't implement the sequence diff --git a/setup.py b/setup.py index e634e37f2..be2f6e211 100755 --- a/setup.py +++ b/setup.py @@ -118,6 +118,7 @@ def get_data_files(self): "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", ],) From b3237e804ae313503f5479349f90066c356b1548 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 12 Jan 2015 20:38:38 +0100 Subject: [PATCH 0305/1392] Bumped version to 0.6.4 --- gitdb/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 2ba8725eb..6554cf904 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 3) +version_info = (0, 6, 4) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index be2f6e211..e4b1b1db1 100755 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ def get_data_files(self): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 3) +version_info = (0, 6, 4) __version__ = '.'.join(str(i) for i in version_info) setup(cmdclass={'build_ext': build_ext_nofail}, From 9aae93ea584c8cf9d1539a60e41c5c37119401d6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Jan 2015 18:16:34 +0100 Subject: [PATCH 0306/1392] Added issuestats to readme file --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 186218d1f..766d9d66a 100644 --- a/README.rst +++ b/README.rst @@ -49,6 +49,12 @@ DEVELOPMENT .. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png :target: https://coveralls.io/r/gitpython-developers/gitdb +.. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/pr + :target: http://www.issuestats.com/github/gitpython-developers/gitdb + +.. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/issue + :target: http://www.issuestats.com/github/gitpython-developers/gitdb + The library is considered mature, and not under active development. It's primary (known) use is in git-python. INFRASTRUCTURE From a8af40edd969d79f1059fef774f5c7d600571999 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Jan 2015 18:18:18 +0100 Subject: [PATCH 0307/1392] Added issuestats badges to readme file --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e15c9bfc5..84b84afd6 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Although memory maps have many advantages, they represent a very limited system [![Build Status](https://travis-ci.org/Byron/smmap.svg?branch=master)](https://travis-ci.org/Byron/smmap) [![Coverage Status](https://coveralls.io/repos/Byron/smmap/badge.png)](https://coveralls.io/r/Byron/smmap) +[![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/pr)](http://www.issuestats.com/github/gitpython-developers/smmap) +[![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/issue)](http://www.issuestats.com/github/gitpython-developers/smmap) Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. From 5a3877a091d16a064a6ec07b1e41e536580831bb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Jan 2015 18:20:30 +0100 Subject: [PATCH 0308/1392] Fixed urls, they changed after moving the repo to gitpython-developers --- README.md | 8 ++++---- doc/source/intro.rst | 4 ++-- setup.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 84b84afd6..1c09ecb89 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ Although memory maps have many advantages, they represent a very limited system ## Overview -[![Build Status](https://travis-ci.org/Byron/smmap.svg?branch=master)](https://travis-ci.org/Byron/smmap) -[![Coverage Status](https://coveralls.io/repos/Byron/smmap/badge.png)](https://coveralls.io/r/Byron/smmap) +[![Build Status](https://travis-ci.org/gitpython-developers/smmap.svg?branch=master)](https://travis-ci.org/gitpython-developers/smmap) +[![Coverage Status](https://coveralls.io/repos/gitpython-developers/smmap/badge.png)](https://coveralls.io/r/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/pr)](http://www.issuestats.com/github/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/issue)](http://www.issuestats.com/github/gitpython-developers/smmap) @@ -63,7 +63,7 @@ It is advised to have a look at the **Usage Guide** for a brief introduction on ## Homepage and Links -The project is home on github at https://github.com/Byron/smmap . +The project is home on github at https://github.com/gitpython-developers/smmap . The latest source can be cloned from github as well: @@ -77,7 +77,7 @@ For support, please use the git-python mailing list: Issues can be filed on github: -* https://github.com/Byron/smmap/issues +* https://github.com/gitpython-developers/smmap/issues ## License Information diff --git a/doc/source/intro.rst b/doc/source/intro.rst index ee3108a6a..15f5bf08a 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -51,7 +51,7 @@ It is advised to have a look at the :ref:`Usage Guide ` for a br ################## Homepage and Links ################## -The project is home on github at `https://github.com/Byron/smmap `_. +The project is home on github at `https://github.com/gitpython-developers/smmap `_. The latest source can be cloned from github as well: @@ -65,7 +65,7 @@ For support, please use the git-python mailing list: Issues can be filed on github: - * https://github.com/Byron/smmap/issues + * https://github.com/gitpython-developers/smmap/issues ################### License Information diff --git a/setup.py b/setup.py index c6afc8960..267dfc7a5 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ if os.path.exists("README.md"): long_description = codecs.open('README.md', "r", "utf-8").read() else: - long_description = "See http://github.com/Byron/smmap" + long_description = "See http://github.com/gitpython-developers/smmap" setup( name="smmap", From 016b58f3a7638d59ee766433649253e2d53e18b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Germ=C3=A1n=20M=2E=20Bravo?= Date: Tue, 7 Apr 2015 08:32:38 -0500 Subject: [PATCH 0309/1392] Duplicate `const` fixed Remove duplicate `const` to stop the warning: "duplicate 'const' declaration specifier" --- gitdb/_delta_apply.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gitdb/_delta_apply.c b/gitdb/_delta_apply.c index 8b0f8e064..f4fffdcce 100644 --- a/gitdb/_delta_apply.c +++ b/gitdb/_delta_apply.c @@ -413,7 +413,7 @@ uint DIV_info_rbound(const DeltaInfoVector* vec, const DeltaInfo* di) // return size of the given delta info item inline -uint DIV_info_size2(const DeltaInfoVector* vec, const DeltaInfo* di, const DeltaInfo const* veclast) +uint DIV_info_size2(const DeltaInfoVector* vec, const DeltaInfo* di, const DeltaInfo* const veclast) { if (veclast == di){ return vec->di_last_size; @@ -534,7 +534,7 @@ uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) } } - const DeltaInfo const* vecend = DIV_end(src); + const DeltaInfo* const vecend = DIV_end(src); const uchar* nstream; for( ;cdi < vecend; ++cdi){ nstream = next_delta_info(src->dstream + cdi->dso, &dc); @@ -753,7 +753,7 @@ PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) PyObject* tmpargs = PyTuple_New(1); const uchar* data = TSI_first(&self->istream); - const uchar const* dend = TSI_end(&self->istream); + const uchar* const dend = TSI_end(&self->istream); DeltaChunk dc; DC_init(&dc, 0, 0, 0, NULL); @@ -979,8 +979,8 @@ PyObject* connect_deltas(PyObject *self, PyObject *dstreams) const uchar* data; Py_ssize_t dlen; PyObject_AsReadBuffer(db, (const void**)&data, &dlen); - const uchar const* dstart = data; - const uchar const* dend = data + dlen; + const uchar* const dstart = data; + const uchar* const dend = data + dlen; div.dstream = dstart; if (dlen > pow(2, 32)){ From 18e4aea23644ea43657cb2e6846b6aaf78720c27 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 8 May 2015 08:56:47 +0200 Subject: [PATCH 0310/1392] fix(tests): remove line failing on power-pc It's worth noting that I never reproduced the issue, nor have I seen a stack-trace. Thus the line is removed in good-faith, but should also not pose any problem considering it was very specific and only in a test-case. The main problem is that I don't understand anymore why that assertion should be true, and thus can't judge the correctness of this fix at all. Closes #25 --- smmap/test/test_util.py | 8 -------- smmap/util.py | 1 - 2 files changed, 9 deletions(-) diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 0a162607e..5a9d7bdf2 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -76,14 +76,6 @@ def test_region(self): assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size) assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) - # with the values we have, this test only works on windows where an alignment - # size of 4096 is assumed. - # We only test on linux as it is inconsitent between the python versions - # as they use different mapping techniques to circumvent the missing offset - # argument of mmap. - if sys.platform != 'win32': - assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) - # END handle platforms # auto-refcount assert rfull.client_count() == 1 diff --git a/smmap/util.py b/smmap/util.py index 1d20e5040..36ff1ab89 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -1,7 +1,6 @@ """Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" import os import sys -import mmap from mmap import mmap, ACCESS_READ try: From 17413029b0f780ac94c24ab2b5f527ded6abdd2a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 22 Aug 2015 16:39:34 +0200 Subject: [PATCH 0311/1392] docs(gitdb): discourage usage of GitDB type --- gitdb/db/git.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gitdb/db/git.py b/gitdb/db/git.py index a4f6f54c3..7a43d7235 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -22,7 +22,11 @@ class GitDB(FileDBBase, ObjectDBW, CompoundDB): """A git-style object database, which contains all objects in the 'objects' - subdirectory""" + subdirectory + + ``IMPORTANT``: The usage of this implementation is highly discouraged as it fails to release file-handles. + This can be a problem with long-running processes and/or big repositories. + """ # Configuration PackDBCls = PackedDB LooseDBCls = LooseObjectDB From 2389b75280efb1a63e6ea578eae7f897fd4beb1b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 4 Oct 2015 19:17:44 +0200 Subject: [PATCH 0312/1392] fix(loose): avoid unnecessary file rename on windows This should workaround possible permission issues. Related to https://github.com/gitpython-developers/GitPython/issues/353 --- gitdb/db/loose.py | 13 +++++++++---- gitdb/ext/smmap | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 4732b56da..1355e1cee 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -226,10 +226,15 @@ def store(self, istream): mkdir(obj_dir) # END handle destination directory # rename onto existing doesn't work on windows - if os.name == 'nt' and isfile(obj_path): - remove(obj_path) - # END handle win322 - rename(tmp_path, obj_path) + if os.name == 'nt': + if isfile(obj_path): + remove(tmp_path) + else: + rename(tmp_path, obj_path) + # end rename only if needed + else: + rename(tmp_path, obj_path) + # END handle win32 # make sure its readable for all ! It started out as rw-- tmp file # but needs to be rwrr diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 84929ed81..18e4aea23 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 84929ed811142e366d6c5916125302c1419acad6 +Subproject commit 18e4aea23644ea43657cb2e6846b6aaf78720c27 From 7a8b138290ccf75da9ec5812dc7b55a40e8eeaeb Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Thu, 15 Oct 2015 09:57:26 +0800 Subject: [PATCH 0313/1392] Add Python 3.5 to test with travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b967f502e..464fafb2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: - 2.7 - 3.3 - 3.4 + - 3.5 install: - pip install coveralls script: From 50d073bd7cf769686efc63e42cb75512952b25ed Mon Sep 17 00:00:00 2001 From: Jesse Weigert Date: Tue, 27 Oct 2015 14:02:20 -0700 Subject: [PATCH 0314/1392] Fix package description --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 267dfc7a5..f6a04827d 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ setup( name="smmap", version=smmap.__version__, - description="A pure git implementation of a sliding window memory map manager", + description="A pure python implementation of a sliding window memory map manager", author=smmap.__author__, author_email=smmap.__contact__, url=smmap.__homepage__, From 490fdc2f27ca91898f09defdf20e1237a5ac12aa Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 30 Nov 2015 08:50:56 +0100 Subject: [PATCH 0315/1392] fix(distribution): remove redundant self-reference --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8a4cd3979..ed4898ec2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -gitdb -smmap>=0.8.3 \ No newline at end of file +smmap>=0.8.3 From d1996e04dbf4841b853b60c1365f0f5fd28d170c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 28 Mar 2016 09:10:31 +0200 Subject: [PATCH 0316/1392] Ignore MANIFEST.in Fixes #25 --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 597944fd4..b939b5ded 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include LICENSE include CHANGES include AUTHORS include README +include MANIFEST.in include gitdb/_fun.c include gitdb/_delta_apply.c From 5ff376161a2f2875d3fc0eb1d9f25027e29460ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Wed, 27 Jul 2016 08:49:52 +0300 Subject: [PATCH 0317/1392] travis: Test with Python 3.5 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 8cab8225f..ba0beaa7b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - "2.7" - "3.3" - "3.4" + - "3.5" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) git: From cadb3d8a095c3e51cede1c33f7dcbf49c1426418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Wed, 27 Jul 2016 08:53:11 +0300 Subject: [PATCH 0318/1392] setup: Add Python 3.5 classifier --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e4b1b1db1..c2f9f1eb1 100755 --- a/setup.py +++ b/setup.py @@ -121,4 +121,5 @@ def get_data_files(self): "Programming Language :: Python :: 3.2", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", ],) From 2af455266cb3ea454c4f38b182825b9e78b8398d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Wed, 27 Jul 2016 09:32:29 +0300 Subject: [PATCH 0319/1392] Spelling fixes --- doc/source/algorithm.rst | 2 +- doc/source/changes.rst | 2 +- gitdb/db/base.py | 2 +- gitdb/db/loose.py | 2 +- gitdb/db/mem.py | 2 +- gitdb/pack.py | 4 ++-- gitdb/stream.py | 2 +- gitdb/test/db/test_loose.py | 2 +- gitdb/test/lib.py | 2 +- gitdb/test/test_pack.py | 2 +- gitdb/test/test_util.py | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/source/algorithm.rst b/doc/source/algorithm.rst index 4374cb820..2e01b3fbe 100644 --- a/doc/source/algorithm.rst +++ b/doc/source/algorithm.rst @@ -92,6 +92,6 @@ Future work Another very promising option is that streaming of delta data is indeed possible. Depending on the configuration of the copy-from-base operations, different optimizations could be applied to reduce the amount of memory required for the final processed delta stream. Some configurations may even allow it to stream data from the base buffer, instead of pre-loading it for random access. -The ability to stream files at reduced memory costs would only be feasible for big files, and would have to be payed with extra pre-processing time. +The ability to stream files at reduced memory costs would only be feasible for big files, and would have to be paid with extra pre-processing time. A very first and simple implementation could avoid memory peaks by streaming the TDS in conjunction with a base buffer, instead of writing everything into a fully allocated target buffer. diff --git a/doc/source/changes.rst b/doc/source/changes.rst index a36fd659c..22deb6db3 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -8,7 +8,7 @@ Changelog * Fixed possibly critical error, see https://github.com/gitpython-developers/GitPython/issues/220 - - However, it only seems to occour on high-entropy data and didn't reoccour after the fix + - However, it only seems to occur on high-entropy data and didn't reoccour after the fix ***** 0.6.0 diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 2615b13cc..2d7b9fa8d 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -177,7 +177,7 @@ def _db_query(self, sha): """:return: database containing the given 20 byte sha :raise BadObject:""" # most databases use binary representations, prevent converting - # it everytime a database is being queried + # it every time a database is being queried try: return self._db_cache[sha] except KeyError: diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 1355e1cee..192c524af 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -174,7 +174,7 @@ def has_object(self, sha): return True except BadObject: return False - # END check existance + # END check existence def store(self, istream): """note: The sha we produce will be hex by nature""" diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 595dbf4fc..871133468 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -98,7 +98,7 @@ def stream_copy(self, sha_iter, odb): for sha in sha_iter: if odb.has_object(sha): continue - # END check object existance + # END check object existence ostream = self.stream(sha) # compressed data including header diff --git a/gitdb/pack.py b/gitdb/pack.py index 511e5571c..2447455b5 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -85,7 +85,7 @@ def pack_object_at(cursor, offset, as_stream): an object of the correct type according to the type_id of the object. If as_stream is True, the object will contain a stream, allowing the data to be read decompressed. - :param data: random accessable data containing all required information + :param data: random accessible data containing all required information :parma offset: offset in to the data at which the object information is located :param as_stream: if True, a stream object will be returned that can read the data, otherwise you receive an info object only""" @@ -447,7 +447,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length): :return: index as in `sha_to_index` or None if the sha was not found in this index file :param partial_bin_sha: an at least two bytes of a partial binary sha as bytes - :param canonical_length: lenght of the original hexadecimal representation of the + :param canonical_length: length of the original hexadecimal representation of the given partial binary sha :raise AmbiguousObjectName:""" if len(partial_bin_sha) < 2: diff --git a/gitdb/stream.py b/gitdb/stream.py index 04dd79f90..be95c113a 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -660,7 +660,7 @@ def __init__(self, fd): def write(self, data): """:raise IOError: If not all bytes could be written - :return: lenght of incoming data""" + :return: length of incoming data""" self.sha1.update(data) cdata = self.zip.compress(data) bytes_written = write(self.fd, cdata) diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 024c194a2..9c25a0249 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -33,4 +33,4 @@ def test_basics(self, path): # END for each sha self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') - # raises if no object could be foudn + # raises if no object could be found diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index a089eac6f..bbdd241cd 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -76,7 +76,7 @@ def wrapper(self, *args, **kwargs): def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the - test suceeds, but leave it otherwise to aid additional debugging""" + test succeeds, but leave it otherwise to aid additional debugging""" def wrapper(self): path = tempfile.mktemp(prefix=func.__name__) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index ff1057237..a8b0b60a4 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -232,7 +232,7 @@ def rewind_streams(): # END verify files exist # END for each packpath, indexpath pair - # verify the packs throughly + # verify the packs thoroughly rewind_streams() entity = PackEntity.create(pack_objs, rw_dir) count = 0 diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 1dee54461..5026f4e62 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -60,7 +60,7 @@ def test_lockedfd(self): self._cmp_contents(my_file, orig_data) assert not os.path.isfile(lockfilepath) - # additional call doesnt fail + # additional call doesn't fail lfd.commit() lfd.rollback() From f957c812ac3221773058ba2fa8cd38017537da8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Wed, 27 Jul 2016 09:34:09 +0300 Subject: [PATCH 0320/1392] Handle more file open/close with "with" --- gitdb/db/ref.py | 3 ++- gitdb/test/db/test_ref.py | 7 +++---- gitdb/test/test_pack.py | 5 ++--- gitdb/test/test_util.py | 10 +++------- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 83a9f611d..2e3db86db 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -41,7 +41,8 @@ def _update_dbs_from_ref_file(self): # try to get as many as possible, don't fail if some are unavailable ref_paths = list() try: - ref_paths = [l.strip() for l in open(self._ref_file, 'r').readlines()] + with open(self._ref_file, 'r') as f: + ref_paths = [l.strip() for l in f] except (OSError, IOError): pass # END handle alternates diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 6bac245c1..20496985e 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -21,10 +21,9 @@ class TestReferenceDB(TestDBBase): def make_alt_file(self, alt_path, alt_list): """Create an alternates file which contains the given alternates. The list can be empty""" - alt_file = open(alt_path, "wb") - for alt in alt_list: - alt_file.write(alt.encode("utf-8") + "\n".encode("ascii")) - alt_file.close() + with open(alt_path, "wb") as alt_file: + for alt in alt_list: + alt_file.write(alt.encode("utf-8") + "\n".encode("ascii")) @with_rw_directory def test_writing(self, path): diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index ff1057237..8ecfcd91a 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -197,7 +197,6 @@ def rewind_streams(): obj.stream.seek(0) # END utility for ppath, ipath, num_obj in zip((pack_path, ) * 2, (index_path, None), (len(pack_objs), None)): - pfile = open(ppath, 'wb') iwrite = None if ipath: ifile = open(ipath, 'wb') @@ -210,8 +209,8 @@ def rewind_streams(): # END rewind streams iteration += 1 - pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) - pfile.close() + with open(ppath, 'wb') as pfile: + pack_sha, index_sha = PackEntity.write_pack(pack_objs, pfile.write, iwrite, object_count=num_obj) assert os.path.getsize(ppath) > 100 # verify pack diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 1dee54461..c2989203e 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -25,19 +25,15 @@ def test_basics(self): def _cmp_contents(self, file_path, data): # raise if data from file at file_path # does not match data string - fp = open(file_path, "rb") - try: + with open(file_path, "rb") as fp: assert fp.read() == data.encode("ascii") - finally: - fp.close() def test_lockedfd(self): my_file = tempfile.mktemp() orig_data = "hello" new_data = "world" - my_file_fp = open(my_file, "wb") - my_file_fp.write(orig_data.encode("ascii")) - my_file_fp.close() + with open(my_file, "wb") as my_file_fp: + my_file_fp.write(orig_data.encode("ascii")) try: lfd = LockedFD(my_file) From 36ed59a1bb44ecada33ce83049605fd7d70e7876 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 7 Sep 2016 14:59:29 +0100 Subject: [PATCH 0321/1392] Support universal wheels --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..3c6e79cf3 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 From fde2dde8e6fe39c8548acb0c919cf18adaf2806a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 12 Sep 2016 20:16:22 +0100 Subject: [PATCH 0322/1392] Do not support universal wheels because you need to build manylinux1/mac/windows wheels instead --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 3c6e79cf3..000000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal=1 From e835f555c2d78601228c73ef048826c1f108cec7 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 1 Oct 2016 21:45:21 +0200 Subject: [PATCH 0323/1392] ci: Enable Appveyor. --- .appveyor.yml | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000..2daadaa4d --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,49 @@ +# CI on Windows via appveyor +environment: + + matrix: + ## MINGW + # + - PYTHON: "C:\\Python27" + PYTHON_VERSION: "2.7" + - PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4" + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5" + - PYTHON: "C:\\Miniconda35-x64" + PYTHON_VERSION: "3.5" + IS_CONDA: "yes" + +install: + - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% + + ## Print configuration for debugging. + # + - | + echo %PATH% + uname -a + where python pip pip2 pip3 pip34 + python --version + python -c "import struct; print(struct.calcsize('P') * 8)" + + - IF "%IS_CONDA%"=="yes" ( + conda info -a & + conda install --yes --quiet pip + ) + - pip install nose wheel coveralls + + ## For commits performed with the default user. + - | + git config --global user.email "travis@ci.com" + git config --global user.name "Travis Runner" + + - pip install -e . + +build: false + +test_script: + - IF "%PYTHON_VERSION%"=="3.5" ( + nosetests -v --with-coverage + ) ELSE ( + nosetests -v + ) From dfadc166beaf28ba216e03e3592649437397fadb Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 1 Oct 2016 22:23:46 +0200 Subject: [PATCH 0324/1392] Appveyor: Add badge. --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1c09ecb89..ef0b23601 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Although memory maps have many advantages, they represent a very limited system ## Overview [![Build Status](https://travis-ci.org/gitpython-developers/smmap.svg?branch=master)](https://travis-ci.org/gitpython-developers/smmap) +[![Build status](https://ci.appveyor.com/api/projects/status/h8rl7thsr42oc0pf?svg=true&passingText=windows%20OK&failingText=windows%20failed)](https://ci.appveyor.com/project/Byron/smmap) [![Coverage Status](https://coveralls.io/repos/gitpython-developers/smmap/badge.png)](https://coveralls.io/r/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/pr)](http://www.issuestats.com/github/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/issue)](http://www.issuestats.com/github/gitpython-developers/smmap) @@ -44,15 +45,15 @@ The package was tested on all of the previously mentioned configurations. [![Documentation Status](https://readthedocs.org/projects/smmap/badge/?version=latest)](https://readthedocs.org/projects/smmap/?badge=latest) Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: - + ```bash $ pip install smmap ``` - + As the command will install smmap in your respective python distribution, you will most likely need root permissions to authorize the required changes. If you have downloaded the source archive, the package can be installed by running the `setup.py` script: - + ```bash $ python setup.py install ``` @@ -68,17 +69,17 @@ The project is home on github at https://github.com/gitpython-developers/smmap . The latest source can be cloned from github as well: * git://github.com/gitpython-developers/smmap.git - - + + For support, please use the git-python mailing list: * http://groups.google.com/group/git-python - + Issues can be filed on github: * https://github.com/gitpython-developers/smmap/issues - + ## License Information From be5d4267f6cc4272e3fbfa1fbd262e984c2077bf Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sun, 2 Oct 2016 01:28:06 +0200 Subject: [PATCH 0325/1392] io: retrofit classes wih destructors into context-mans --- smmap/buf.py | 12 +- smmap/mman.py | 6 + smmap/test/lib.py | 13 +- smmap/test/test_buf.py | 202 +++++++++++------------ smmap/test/test_mman.py | 317 ++++++++++++++++++------------------ smmap/test/test_tutorial.py | 121 +++++++------- smmap/test/test_util.py | 48 +++--- 7 files changed, 368 insertions(+), 351 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index e6f24341d..438292b60 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -47,6 +47,12 @@ def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0): def __del__(self): self.end_access() + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.end_access() + def __len__(self): return self._size @@ -83,7 +89,7 @@ def __getslice__(self, i, j): # in the previous iteration of this code pyvers = sys.version_info[:2] if (3, 0) <= pyvers <= (3, 3): - # Memory view cannot be joined below python 3.4 ... + # Memory view cannot be joined below python 3.4 ... out = bytes() while l: c.use_region(ofs, l) @@ -91,7 +97,7 @@ def __getslice__(self, i, j): d = c.buffer()[:l] ofs += len(d) l -= len(d) - # This is slower than the join ... but what can we do ... + # This is slower than the join ... but what can we do ... out += d del(d) # END while there are bytes to read @@ -104,7 +110,7 @@ def __getslice__(self, i, j): d = c.buffer()[:l] ofs += len(d) l -= len(d) - # Make sure we don't keep references, as c.use_region() might attempt to free resources, but + # Make sure we don't keep references, as c.use_region() might attempt to free resources, but # can't unless we use pure bytes if hasattr(d, 'tobytes'): d = d.tobytes() diff --git a/smmap/mman.py b/smmap/mman.py index 7180c75b3..af12ee98f 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -46,6 +46,12 @@ def __init__(self, manager=None, regions=None): def __del__(self): self._destroy() + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self._destroy() + def _destroy(self): """Destruction code to decrement counters""" self.unuse_region() diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 93cb09a5e..f86c0c6f1 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -21,10 +21,9 @@ def __init__(self, size, prefix=''): self._path = tempfile.mktemp(prefix=prefix) self._size = size - fp = open(self._path, "wb") - fp.seek(size - 1) - fp.write(b'1') - fp.close() + with open(self._path, "wb") as fp: + fp.seek(size - 1) + fp.write(b'1') assert os.path.getsize(self.path) == size @@ -35,6 +34,12 @@ def __del__(self): pass # END exception handling + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.__del__() + @property def path(self): return self._path diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 984b43254..3b6009e11 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -25,104 +25,104 @@ class TestBuf(TestBase): def test_basics(self): - fc = FileCreator(self.k_window_test_size, "buffer_test") - - # invalid paths fail upon construction - c = man_optimal.make_cursor(fc.path) - self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor - self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large - - buf = SlidingWindowMapBuffer() # can create uninitailized buffers - assert buf.cursor() is None - - # can call end access any time - buf.end_access() - buf.end_access() - assert len(buf) == 0 - - # begin access can revive it, if the offset is suitable - offset = 100 - assert buf.begin_access(c, fc.size) == False - assert buf.begin_access(c, offset) == True - assert len(buf) == fc.size - offset - assert buf.cursor().is_valid() - - # empty begin access keeps it valid on the same path, but alters the offset - assert buf.begin_access() == True - assert len(buf) == fc.size - assert buf.cursor().is_valid() - - # simple access - with open(fc.path, 'rb') as fp: - data = fp.read() - assert data[offset] == buf[0] - assert data[offset:offset * 2] == buf[0:offset] - - # negative indices, partial slices - assert buf[-1] == buf[len(buf) - 1] - assert buf[-10:] == buf[len(buf) - 10:len(buf)] - - # end access makes its cursor invalid - buf.end_access() - assert not buf.cursor().is_valid() - assert buf.cursor().is_associated() # but it remains associated - - # an empty begin access fixes it up again - assert buf.begin_access() == True and buf.cursor().is_valid() - del(buf) # ends access automatically - del(c) - - assert man_optimal.num_file_handles() == 1 - - # PERFORMANCE - # blast away with random access and a full mapping - we don't want to - # exaggerate the manager's overhead, but measure the buffer overhead - # We do it once with an optimal setting, and with a worse manager which - # will produce small mappings only ! - max_num_accesses = 100 - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - for manager, man_id in ((man_optimal, 'optimal'), - (man_worst_case, 'worst case'), - (static_man, 'static optimal')): - buf = SlidingWindowMapBuffer(manager.make_cursor(item)) - assert manager.num_file_handles() == 1 - for access_mode in range(2): # single, multi - num_accesses_left = max_num_accesses - num_bytes = 0 - fsize = fc.size - - st = time() - buf.begin_access() - while num_accesses_left: - num_accesses_left -= 1 - if access_mode: # multi - ofs_start = randint(0, fsize) - ofs_end = randint(ofs_start, fsize) - d = buf[ofs_start:ofs_end] - assert len(d) == ofs_end - ofs_start - assert d == data[ofs_start:ofs_end] - num_bytes += len(d) - del d - else: - pos = randint(0, fsize) - assert buf[pos] == data[pos] - num_bytes += 1 - # END handle mode - # END handle num accesses - - buf.end_access() - assert manager.num_file_handles() - assert manager.collect() - assert manager.num_file_handles() == 0 - elapsed = max(time() - st, 0.001) # prevent zero division errors on windows - mb = float(1000 * 1000) - mode_str = (access_mode and "slice") or "single byte" - print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" - % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), - file=sys.stderr) - # END handle access mode - del buf - # END for each manager - # END for each input - os.close(fd) + with FileCreator(self.k_window_test_size, "buffer_test") as fc: + + # invalid paths fail upon construction + c = man_optimal.make_cursor(fc.path) + self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor + self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large + + buf = SlidingWindowMapBuffer() # can create uninitailized buffers + assert buf.cursor() is None + + # can call end access any time + buf.end_access() + buf.end_access() + assert len(buf) == 0 + + # begin access can revive it, if the offset is suitable + offset = 100 + assert buf.begin_access(c, fc.size) == False + assert buf.begin_access(c, offset) == True + assert len(buf) == fc.size - offset + assert buf.cursor().is_valid() + + # empty begin access keeps it valid on the same path, but alters the offset + assert buf.begin_access() == True + assert len(buf) == fc.size + assert buf.cursor().is_valid() + + # simple access + with open(fc.path, 'rb') as fp: + data = fp.read() + assert data[offset] == buf[0] + assert data[offset:offset * 2] == buf[0:offset] + + # negative indices, partial slices + assert buf[-1] == buf[len(buf) - 1] + assert buf[-10:] == buf[len(buf) - 10:len(buf)] + + # end access makes its cursor invalid + buf.end_access() + assert not buf.cursor().is_valid() + assert buf.cursor().is_associated() # but it remains associated + + # an empty begin access fixes it up again + assert buf.begin_access() == True and buf.cursor().is_valid() + del(buf) # ends access automatically + del(c) + + assert man_optimal.num_file_handles() == 1 + + # PERFORMANCE + # blast away with random access and a full mapping - we don't want to + # exaggerate the manager's overhead, but measure the buffer overhead + # We do it once with an optimal setting, and with a worse manager which + # will produce small mappings only ! + max_num_accesses = 100 + fd = os.open(fc.path, os.O_RDONLY) + for item in (fc.path, fd): + for manager, man_id in ((man_optimal, 'optimal'), + (man_worst_case, 'worst case'), + (static_man, 'static optimal')): + buf = SlidingWindowMapBuffer(manager.make_cursor(item)) + assert manager.num_file_handles() == 1 + for access_mode in range(2): # single, multi + num_accesses_left = max_num_accesses + num_bytes = 0 + fsize = fc.size + + st = time() + buf.begin_access() + while num_accesses_left: + num_accesses_left -= 1 + if access_mode: # multi + ofs_start = randint(0, fsize) + ofs_end = randint(ofs_start, fsize) + d = buf[ofs_start:ofs_end] + assert len(d) == ofs_end - ofs_start + assert d == data[ofs_start:ofs_end] + num_bytes += len(d) + del d + else: + pos = randint(0, fsize) + assert buf[pos] == data[pos] + num_bytes += 1 + # END handle mode + # END handle num accesses + + buf.end_access() + assert manager.num_file_handles() + assert manager.collect() + assert manager.num_file_handles() == 0 + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows + mb = float(1000 * 1000) + mode_str = (access_mode and "slice") or "single byte" + print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" + % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), + file=sys.stderr) + # END handle access mode + del buf + # END for each manager + # END for each input + os.close(fd) diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index c8b9c703e..96bc355b7 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -19,19 +19,18 @@ class TestMMan(TestBase): def test_cursor(self): - fc = FileCreator(self.k_window_test_size, "cursor_test") - - man = SlidingWindowMapManager() - ci = WindowCursor(man) # invalid cursor - assert not ci.is_valid() - assert not ci.is_associated() - assert ci.size() == 0 # this is cached, so we can query it in invalid state - - cv = man.make_cursor(fc.path) - assert not cv.is_valid() # no region mapped yet - assert cv.is_associated() # but it know where to map it from - assert cv.file_size() == fc.size - assert cv.path() == fc.path + with FileCreator(self.k_window_test_size, "cursor_test") as fc: + man = SlidingWindowMapManager() + ci = WindowCursor(man) # invalid cursor + assert not ci.is_valid() + assert not ci.is_associated() + assert ci.size() == 0 # this is cached, so we can query it in invalid state + + cv = man.make_cursor(fc.path) + assert not cv.is_valid() # no region mapped yet + assert cv.is_associated() # but it know where to map it from + assert cv.file_size() == fc.size + assert cv.path() == fc.path # copy module cio = copy(cv) @@ -74,150 +73,154 @@ def test_memory_manager(self): assert man._collect_lru_region(sys.maxsize) == 0 # use a region, verify most basic functionality - fc = FileCreator(self.k_window_test_size, "manager_test") - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - c = man.make_cursor(item) - assert c.path_or_fd() is item - assert c.use_region(10, 10).is_valid() - assert c.ofs_begin() == 10 - assert c.size() == 10 - with open(fc.path, 'rb') as fp: - assert c.buffer()[:] == fp.read(20)[10:] - - if isinstance(item, int): - self.assertRaises(ValueError, c.path) - else: - self.assertRaises(ValueError, c.fd) - # END handle value error - # END for each input - os.close(fd) - # END for each manager type + with FileCreator(self.k_window_test_size, "manager_test") as fc: + fd = os.open(fc.path, os.O_RDONLY) + try: + for item in (fc.path, fd): + c = man.make_cursor(item) + assert c.path_or_fd() is item + assert c.use_region(10, 10).is_valid() + assert c.ofs_begin() == 10 + assert c.size() == 10 + with open(fc.path, 'rb') as fp: + assert c.buffer()[:] == fp.read(20)[10:] + + if isinstance(item, int): + self.assertRaises(ValueError, c.path) + else: + self.assertRaises(ValueError, c.fd) + # END handle value error + # END for each input + finally: + os.close(fd) + # END for each manasger type def test_memman_operation(self): # test more access, force it to actually unmap regions - fc = FileCreator(self.k_window_test_size, "manager_operation_test") - with open(fc.path, 'rb') as fp: - data = fp.read() - fd = os.open(fc.path, os.O_RDONLY) - max_num_handles = 15 - # small_size = - for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), - (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): - for item in (fc.path, fd): - assert len(data) == fc.size - - # small windows, a reasonable max memory. Not too many regions at once - man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) - c = man.make_cursor(item) - - # still empty (more about that is tested in test_memory_manager() - assert man.num_open_files() == 0 - assert man.mapped_memory_size() == 0 - - base_offset = 5000 - # window size is 0 for static managers, hence size will be 0. We take that into consideration - size = man.window_size() // 2 - assert c.use_region(base_offset, size).is_valid() - rr = c.region() - assert rr.client_count() == 2 # the manager and the cursor and us - - assert man.num_open_files() == 1 - assert man.num_file_handles() == 1 - assert man.mapped_memory_size() == rr.size() - - # assert c.size() == size # the cursor may overallocate in its static version - assert c.ofs_begin() == base_offset - assert rr.ofs_begin() == 0 # it was aligned and expanded - if man.window_size(): - # but isn't larger than the max window (aligned) - assert rr.size() == align_to_mmap(man.window_size(), True) - else: - assert rr.size() == fc.size - # END ignore static managers which dont use windows and are aligned to file boundaries - - assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] - - # obtain second window, which spans the first part of the file - it is a still the same window - nsize = (size or fc.size) - 10 - assert c.use_region(0, nsize).is_valid() - assert c.region() == rr - assert man.num_file_handles() == 1 - assert c.size() == nsize - assert c.ofs_begin() == 0 - assert c.buffer()[:] == data[:nsize] - - # map some part at the end, our requested size cannot be kept - overshoot = 4000 - base_offset = fc.size - (size or c.size()) + overshoot - assert c.use_region(base_offset, size).is_valid() - if man.window_size(): - assert man.num_file_handles() == 2 - assert c.size() < size - assert c.region() is not rr # old region is still available, but has not curser ref anymore - assert rr.client_count() == 1 # only held by manager - else: - assert c.size() < fc.size - # END ignore static managers which only have one handle per file - rr = c.region() - assert rr.client_count() == 2 # manager + cursor - assert rr.ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr.ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] - - # unising a region makes the cursor invalid - c.unuse_region() - assert not c.is_valid() - if man.window_size(): - # but doesn't change anything regarding the handle count - we cache it and only - # remove mapped regions if we have to - assert man.num_file_handles() == 2 - # END ignore this for static managers - - # iterate through the windows, verify data contents - # this will trigger map collection after a while - max_random_accesses = 5000 - num_random_accesses = max_random_accesses - memory_read = 0 - st = time() - - # cache everything to get some more performance - includes_ofs = c.includes_ofs - max_mapped_memory_size = man.max_mapped_memory_size() - max_file_handles = man.max_file_handles() - mapped_memory_size = man.mapped_memory_size - num_file_handles = man.num_file_handles - while num_random_accesses: - num_random_accesses -= 1 - base_offset = randint(0, fc.size - 1) - - # precondition - if man.window_size(): - assert max_mapped_memory_size >= mapped_memory_size() - # END statics will overshoot, which is fine - assert max_file_handles >= num_file_handles() - assert c.use_region(base_offset, (size or c.size())).is_valid() - csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset + csize] - memory_read += csize - - assert includes_ofs(base_offset) - assert includes_ofs(base_offset + csize - 1) - assert not includes_ofs(base_offset + csize) - # END while we should do an access - elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows - mb = float(1000 * 1000) - print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed), - file=sys.stderr) - - # an offset as large as the size doesn't work ! - assert not c.use_region(fc.size, size).is_valid() - - # collection - it should be able to collect all - assert man.num_file_handles() - assert man.collect() - assert man.num_file_handles() == 0 - # END for each item - # END for each manager type - os.close(fd) + with FileCreator(self.k_window_test_size, "manager_operation_test") as fc: + with open(fc.path, 'rb') as fp: + data = fp.read() + fd = os.open(fc.path, os.O_RDONLY) + try: + max_num_handles = 15 + # small_size = + for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), + (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): + for item in (fc.path, fd): + assert len(data) == fc.size + + # small windows, a reasonable max memory. Not too many regions at once + man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2]) + c = man.make_cursor(item) + + # still empty (more about that is tested in test_memory_manager() + assert man.num_open_files() == 0 + assert man.mapped_memory_size() == 0 + + base_offset = 5000 + # window size is 0 for static managers, hence size will be 0. We take that into consideration + size = man.window_size() // 2 + assert c.use_region(base_offset, size).is_valid() + rr = c.region() + assert rr.client_count() == 2 # the manager and the cursor and us + + assert man.num_open_files() == 1 + assert man.num_file_handles() == 1 + assert man.mapped_memory_size() == rr.size() + + # assert c.size() == size # the cursor may overallocate in its static version + assert c.ofs_begin() == base_offset + assert rr.ofs_begin() == 0 # it was aligned and expanded + if man.window_size(): + # but isn't larger than the max window (aligned) + assert rr.size() == align_to_mmap(man.window_size(), True) + else: + assert rr.size() == fc.size + # END ignore static managers which dont use windows and are aligned to file boundaries + + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] + + # obtain second window, which spans the first part of the file - it is a still the same window + nsize = (size or fc.size) - 10 + assert c.use_region(0, nsize).is_valid() + assert c.region() == rr + assert man.num_file_handles() == 1 + assert c.size() == nsize + assert c.ofs_begin() == 0 + assert c.buffer()[:] == data[:nsize] + + # map some part at the end, our requested size cannot be kept + overshoot = 4000 + base_offset = fc.size - (size or c.size()) + overshoot + assert c.use_region(base_offset, size).is_valid() + if man.window_size(): + assert man.num_file_handles() == 2 + assert c.size() < size + assert c.region() is not rr # old region is still available, but has not curser ref anymore + assert rr.client_count() == 1 # only held by manager + else: + assert c.size() < fc.size + # END ignore static managers which only have one handle per file + rr = c.region() + assert rr.client_count() == 2 # manager + cursor + assert rr.ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr.ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] + + # unising a region makes the cursor invalid + c.unuse_region() + assert not c.is_valid() + if man.window_size(): + # but doesn't change anything regarding the handle count - we cache it and only + # remove mapped regions if we have to + assert man.num_file_handles() == 2 + # END ignore this for static managers + + # iterate through the windows, verify data contents + # this will trigger map collection after a while + max_random_accesses = 5000 + num_random_accesses = max_random_accesses + memory_read = 0 + st = time() + + # cache everything to get some more performance + includes_ofs = c.includes_ofs + max_mapped_memory_size = man.max_mapped_memory_size() + max_file_handles = man.max_file_handles() + mapped_memory_size = man.mapped_memory_size + num_file_handles = man.num_file_handles + while num_random_accesses: + num_random_accesses -= 1 + base_offset = randint(0, fc.size - 1) + + # precondition + if man.window_size(): + assert max_mapped_memory_size >= mapped_memory_size() + # END statics will overshoot, which is fine + assert max_file_handles >= num_file_handles() + assert c.use_region(base_offset, (size or c.size())).is_valid() + csize = c.size() + assert c.buffer()[:] == data[base_offset:base_offset + csize] + memory_read += csize + + assert includes_ofs(base_offset) + assert includes_ofs(base_offset + csize - 1) + assert not includes_ofs(base_offset + csize) + # END while we should do an access + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows + mb = float(1000 * 1000) + print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" + % (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed), + file=sys.stderr) + + # an offset as large as the size doesn't work ! + assert not c.use_region(fc.size, size).is_valid() + + # collection - it should be able to collect all + assert man.num_file_handles() + assert man.collect() + assert man.num_file_handles() == 0 + # END for each item + # END for each manager type + finally: + os.close(fd) diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index f7a2128a9..b03db9be2 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -20,65 +20,62 @@ def test_example(self): # Cursors ########## import smmap.test.lib - fc = smmap.test.lib.FileCreator(1024 * 1024 * 8, "test_file") - - # obtain a cursor to access some file. - c = mman.make_cursor(fc.path) - - # the cursor is now associated with the file, but not yet usable - assert c.is_associated() - assert not c.is_valid() - - # before you can use the cursor, you have to specify a window you want to - # access. The following just says you want as much data as possible starting - # from offset 0. - # To be sure your region could be mapped, query for validity - assert c.use_region().is_valid() # use_region returns self - - # once a region was mapped, you must query its dimension regularly - # to assure you don't try to access its buffer out of its bounds - assert c.size() - c.buffer()[0] # first byte - c.buffer()[1:10] # first 9 bytes - c.buffer()[c.size() - 1] # last byte - - # its recommended not to create big slices when feeding the buffer - # into consumers (e.g. struct or zlib). - # Instead, either give the buffer directly, or use pythons buffer command. - from smmap.util import buffer - buffer(c.buffer(), 1, 9) # first 9 bytes without copying them - - # you can query absolute offsets, and check whether an offset is included - # in the cursor's data. - assert c.ofs_begin() < c.ofs_end() - assert c.includes_ofs(100) - - # If you are over out of bounds with one of your region requests, the - # cursor will be come invalid. It cannot be used in that state - assert not c.use_region(fc.size, 100).is_valid() - # map as much as possible after skipping the first 100 bytes - assert c.use_region(100).is_valid() - - # You can explicitly free cursor resources by unusing the cursor's region - c.unuse_region() - assert not c.is_valid() - - # Buffers - ######### - # Create a default buffer which can operate on the whole file - buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) - - # you can use it right away - assert buf.cursor().is_valid() - - buf[0] # access the first byte - buf[-1] # access the last ten bytes on the file - buf[-10:] # access the last ten bytes - - # If you want to keep the instance between different accesses, use the - # dedicated methods - buf.end_access() - assert not buf.cursor().is_valid() # you cannot use the buffer anymore - assert buf.begin_access(offset=10) # start using the buffer at an offset - - # it will stop using resources automatically once it goes out of scope + with smmap.test.lib.FileCreator(1024 * 1024 * 8, "test_file") as fc: + # obtain a cursor to access some file. + c = mman.make_cursor(fc.path) + + # the cursor is now associated with the file, but not yet usable + assert c.is_associated() + assert not c.is_valid() + + # before you can use the cursor, you have to specify a window you want to + # access. The following just says you want as much data as possible starting + # from offset 0. + # To be sure your region could be mapped, query for validity + assert c.use_region().is_valid() # use_region returns self + + # once a region was mapped, you must query its dimension regularly + # to assure you don't try to access its buffer out of its bounds + assert c.size() + c.buffer()[0] # first byte + c.buffer()[1:10] # first 9 bytes + c.buffer()[c.size() - 1] # last byte + + # its recommended not to create big slices when feeding the buffer + # into consumers (e.g. struct or zlib). + # Instead, either give the buffer directly, or use pythons buffer command. + from smmap.util import buffer + buffer(c.buffer(), 1, 9) # first 9 bytes without copying them + + # you can query absolute offsets, and check whether an offset is included + # in the cursor's data. + assert c.ofs_begin() < c.ofs_end() + assert c.includes_ofs(100) + + # If you are over out of bounds with one of your region requests, the + # cursor will be come invalid. It cannot be used in that state + assert not c.use_region(fc.size, 100).is_valid() + # map as much as possible after skipping the first 100 bytes + assert c.use_region(100).is_valid() + + # You can explicitly free cursor resources by unusing the cursor's region + c.unuse_region() + assert not c.is_valid() + + # Buffers + ######### + # Create a default buffer which can operate on the whole file + buf = smmap.SlidingWindowMapBuffer(mman.make_cursor(fc.path)) + + # you can use it right away + assert buf.cursor().is_valid() + + buf[0] # access the first byte + buf[-1] # access the last ten bytes on the file + buf[-10:] # access the last ten bytes + + # If you want to keep the instance between different accesses, use the + # dedicated methods + buf.end_access() + assert not buf.cursor().is_valid() # you cannot use the buffer anymore + assert buf.begin_access(offset=10) # start using the buffer at an offset diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 5a9d7bdf2..e6ac10f35 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -60,22 +60,22 @@ def test_window(self): assert wc.ofs == 0 and wc.size == align_to_mmap(wc.size, True) def test_region(self): - fc = FileCreator(self.k_window_test_size, "window_test") - half_size = fc.size // 2 - rofs = align_to_mmap(4200, False) - rfull = MapRegion(fc.path, 0, fc.size) - rhalfofs = MapRegion(fc.path, rofs, fc.size) - rhalfsize = MapRegion(fc.path, 0, half_size) + with FileCreator(self.k_window_test_size, "window_test") as fc: + half_size = fc.size // 2 + rofs = align_to_mmap(4200, False) + rfull = MapRegion(fc.path, 0, fc.size) + rhalfofs = MapRegion(fc.path, rofs, fc.size) + rhalfsize = MapRegion(fc.path, 0, half_size) - # offsets - assert rfull.ofs_begin() == 0 and rfull.size() == fc.size - assert rfull.ofs_end() == fc.size # if this method works, it works always + # offsets + assert rfull.ofs_begin() == 0 and rfull.size() == fc.size + assert rfull.ofs_end() == fc.size # if this method works, it works always - assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs - assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size + assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs + assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size - assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size) - assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size) + assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) # auto-refcount assert rfull.client_count() == 1 @@ -87,17 +87,17 @@ def test_region(self): assert w.ofs == rfull.ofs_begin() and w.ofs_end() == rfull.ofs_end() def test_region_list(self): - fc = FileCreator(100, "sample_file") - - fd = os.open(fc.path, os.O_RDONLY) - for item in (fc.path, fd): - ml = MapRegionList(item) - - assert len(ml) == 0 - assert ml.path_or_fd() == item - assert ml.file_size() == fc.size - # END handle input - os.close(fd) + with FileCreator(100, "sample_file") as fc: + fd = os.open(fc.path, os.O_RDONLY) + try: + for item in (fc.path, fd): + ml = MapRegionList(item) + + assert len(ml) == 0 + assert ml.path_or_fd() == item + assert ml.file_size() == fc.size + finally: + os.close(fd) def test_util(self): assert isinstance(is_64_bit(), bool) # just call it From 62815b5c5a4c39e9ace3d20ec0c593011201dbcf Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 4 Oct 2016 18:04:24 +0100 Subject: [PATCH 0326/1392] support optional gitdb_speedups --- README.rst | 7 + gitdb/_delta_apply.c | 1154 ------------------------------------------ gitdb/_delta_apply.h | 6 - gitdb/_fun.c | 107 ---- gitdb/fun.py | 2 +- gitdb/pack.py | 2 +- gitdb/stream.py | 2 +- setup.cfg | 2 + setup.py | 154 ++---- 9 files changed, 49 insertions(+), 1387 deletions(-) delete mode 100644 gitdb/_delta_apply.c delete mode 100644 gitdb/_delta_apply.h delete mode 100644 gitdb/_fun.c create mode 100644 setup.cfg diff --git a/README.rst b/README.rst index 766d9d66a..1b754d09b 100644 --- a/README.rst +++ b/README.rst @@ -20,6 +20,13 @@ From `PyPI `_ pip install gitdb +SPEEDUPS +======== + +If you want to go up to 20% faster, you can install gitdb-speedups with: + + pip install gitdb-speedups + REQUIREMENTS ============ diff --git a/gitdb/_delta_apply.c b/gitdb/_delta_apply.c deleted file mode 100644 index f4fffdcce..000000000 --- a/gitdb/_delta_apply.c +++ /dev/null @@ -1,1154 +0,0 @@ -#include <_delta_apply.h> -#include -#include -#include -#include -#include - - - -typedef unsigned long long ull; -typedef unsigned int uint; -typedef unsigned char uchar; -typedef unsigned short ushort; -typedef uchar bool; - -// Constants -const ull gDIV_grow_by = 100; - - - -// DELTA STREAM ACCESS -/////////////////////// -inline -ull msb_size(const uchar** datap, const uchar* top) -{ - const uchar *data = *datap; - ull cmd, size = 0; - uint i = 0; - do { - cmd = *data++; - size |= (cmd & 0x7f) << i; - i += 7; - } while (cmd & 0x80 && data < top); - *datap = data; - return size; -} - - -// TOP LEVEL STREAM INFO -///////////////////////////// -typedef struct { - const uchar *tds; // Toplevel delta stream - const uchar *cstart; // start of the chunks - Py_ssize_t tdslen; // size of tds in bytes - Py_ssize_t target_size; // size of the target buffer which can hold all data - uint num_chunks; // amount of chunks in the delta stream - PyObject *parent_object; -} ToplevelStreamInfo; - - -void TSI_init(ToplevelStreamInfo* info) -{ - info->tds = NULL; - info->cstart = NULL; - info->tdslen = 0; - info->num_chunks = 0; - info->target_size = 0; - info->parent_object = 0; -} - -void TSI_destroy(ToplevelStreamInfo* info) -{ -#ifdef DEBUG - fprintf(stderr, "TSI_destroy: %p\n", info); -#endif - - if (info->parent_object){ - Py_DECREF(info->parent_object); - info->parent_object = NULL; - } else if (info->tds){ - PyMem_Free((void*)info->tds); - } - info->tds = NULL; - info->cstart = NULL; - info->tdslen = 0; - info->num_chunks = 0; -} - -inline -const uchar* TSI_end(ToplevelStreamInfo* info) -{ - return info->tds + info->tdslen; -} - -inline -const uchar* TSI_first(ToplevelStreamInfo* info) -{ - return info->cstart; -} - -// set the stream, and initialize it -// initialize our set stream to point to the first chunk -// Fill in the header information, which is the base and target size -inline -void TSI_set_stream(ToplevelStreamInfo* info, const uchar* stream) -{ - info->tds = stream; - info->cstart = stream; - - assert(info->tds && info->tdslen); - - // init stream - const uchar* tdsend = TSI_end(info); - msb_size(&info->cstart, tdsend); // base size - info->target_size = msb_size(&info->cstart, tdsend); -} - - - -// duplicate the data currently owned by the parent object drop its refcount -// return 1 on success -bool TSI_copy_stream_from_object(ToplevelStreamInfo* info) -{ - assert(info->parent_object); - - uchar* ptmp = PyMem_Malloc(info->tdslen); - if (!ptmp){ - return 0; - } - uint ofs = (uint)(info->cstart - info->tds); - memcpy((void*)ptmp, info->tds, info->tdslen); - - info->tds = ptmp; - info->cstart = ptmp + ofs; - - Py_DECREF(info->parent_object); - info->parent_object = 0; - - return 1; -} - -// Transfer ownership of the given stream into our instance. The amount of chunks -// remains the same, and needs to be set by the caller -void TSI_replace_stream(ToplevelStreamInfo* info, const uchar* stream, uint streamlen) -{ - assert(info->parent_object == 0); - - uint ofs = (uint)(info->cstart - info->tds); - if (info->tds){ - PyMem_Free((void*)info->tds); - } - info->tds = stream; - info->cstart = info->tds + ofs; - info->tdslen = streamlen; - -} - -// DELTA CHUNK -//////////////// -// Internal Delta Chunk Objects -// They are just used to keep information parsed from a stream -// The data pointer is always shared -typedef struct { - ull to; - uint ts; - uint so; - const uchar* data; -} DeltaChunk; - -// forward declarations -const uchar* next_delta_info(const uchar*, DeltaChunk*); - -inline -void DC_init(DeltaChunk* dc, ull to, ull ts, ull so, const uchar* data) -{ - dc->to = to; - dc->ts = ts; - dc->so = so; - dc->data = NULL; -} - - -inline -ull DC_rbound(const DeltaChunk* dc) -{ - return dc->to + dc->ts; -} - -inline -void DC_print(const DeltaChunk* dc, const char* prefix) -{ - fprintf(stderr, "%s-dc: to = %i, ts = %i, so = %i, data = %p\n", prefix, (int)dc->to, dc->ts, dc->so, dc->data); -} - -// Apply -inline -void DC_apply(const DeltaChunk* dc, const uchar* base, PyObject* writer, PyObject* tmpargs) -{ - PyObject* buffer = 0; - if (dc->data){ - buffer = PyBuffer_FromMemory((void*)dc->data, dc->ts); - } else { - buffer = PyBuffer_FromMemory((void*)(base + dc->so), dc->ts); - } - - if (PyTuple_SetItem(tmpargs, 0, buffer)){ - assert(0); - } - - - // tuple steals reference, and will take care about the deallocation - PyObject_Call(writer, tmpargs, NULL); - -} - -// Encode the information in the given delta chunk and write the byte-stream -// into the given output stream -// It will be copied into the given bounds, the given size must be the final size -// and work with the given relative offset - hence the bounds are assumed to be -// correct and to fit within the unaltered dc -inline -void DC_encode_to(const DeltaChunk* dc, uchar** pout, uint ofs, uint size) -{ - uchar* out = *pout; - if (dc->data){ - *out++ = (uchar)size; - memcpy(out, dc->data+ofs, size); - out += size; - } else { - uchar i = 0x80; - uchar* op = out++; - uint moff = dc->so+ofs; - - if (moff & 0x000000ff) - *out++ = moff >> 0, i |= 0x01; - if (moff & 0x0000ff00) - *out++ = moff >> 8, i |= 0x02; - if (moff & 0x00ff0000) - *out++ = moff >> 16, i |= 0x04; - if (moff & 0xff000000) - *out++ = moff >> 24, i |= 0x08; - - if (size & 0x00ff) - *out++ = size >> 0, i |= 0x10; - if (size & 0xff00) - *out++ = size >> 8, i |= 0x20; - - *op = i; - } - - *pout = out; -} - -// Return: amount of bytes one would need to encode dc -inline -ushort DC_count_encode_bytes(const DeltaChunk* dc) -{ - if (dc->data){ - return 1 + dc->ts; // cmd byte + actual data bytes - } else { - ushort c = 1; // cmd byte - uint ts = dc->ts; - ull so = dc->so; - - // offset - c += (so & 0x000000FF) > 0; - c += (so & 0x0000FF00) > 0; - c += (so & 0x00FF0000) > 0; - c += (so & 0xFF000000) > 0; - - // size - max size is 0x10000, its encoded with 0 size bits - c += (ts & 0x000000FF) > 0; - c += (ts & 0x0000FF00) > 0; - - return c; - } -} - - - -// DELTA INFO -///////////// -typedef struct { - uint dso; // delta stream offset, relative to the very start of the stream - uint to; // target offset (cache) -} DeltaInfo; - - -// DELTA INFO VECTOR -////////////////////// - -typedef struct { - DeltaInfo *mem; // Memory for delta infos - uint di_last_size; // size of the last element - we can't compute it using the next bound - const uchar *dstream; // borrowed ointer to delta stream we index - Py_ssize_t size; // Amount of DeltaInfos - Py_ssize_t reserved_size; // Reserved amount of DeltaInfos -} DeltaInfoVector; - - - -// Reserve enough memory to hold the given amount of delta chunks -// Return 1 on success -// NOTE: added a minimum allocation to assure reallocation is not done -// just for a single additional entry. DIVs change often, and reallocs are expensive -inline -int DIV_reserve_memory(DeltaInfoVector* vec, uint num_dc) -{ - if (num_dc <= vec->reserved_size){ - return 1; - } - -#ifdef DEBUG - bool was_null = vec->mem == NULL; -#endif - - if (vec->mem == NULL){ - vec->mem = PyMem_Malloc(num_dc * sizeof(DeltaInfo)); - } else { - vec->mem = PyMem_Realloc(vec->mem, num_dc * sizeof(DeltaInfo)); - } - - if (vec->mem == NULL){ - Py_FatalError("Could not allocate memory for append operation"); - } - - vec->reserved_size = num_dc; - -#ifdef DEBUG - const char* format = "Allocated %i bytes at %p, to hold up to %i chunks\n"; - if (!was_null) - format = "Re-allocated %i bytes at %p, to hold up to %i chunks\n"; - fprintf(stderr, format, (int)(vec->reserved_size * sizeof(DeltaInfo)), vec->mem, (int)vec->reserved_size); -#endif - - return vec->mem != NULL; -} - -/* -Grow the delta chunk list by the given amount of bytes. -This may trigger a realloc, but will do nothing if the reserved size is already -large enough. -Return 1 on success, 0 on failure -*/ -inline -int DIV_grow_by(DeltaInfoVector* vec, uint num_dc) -{ - return DIV_reserve_memory(vec, vec->reserved_size + num_dc); -} - -int DIV_init(DeltaInfoVector* vec, ull initial_size) -{ - vec->mem = NULL; - vec->dstream = NULL; - vec->size = 0; - vec->reserved_size = 0; - vec->di_last_size = 0; - - return DIV_grow_by(vec, initial_size); -} - -inline -Py_ssize_t DIV_len(const DeltaInfoVector* vec) -{ - return vec->size; -} - -inline -uint DIV_lbound(const DeltaInfoVector* vec) -{ - assert(vec->size && vec->mem); - return vec->mem->to; -} - -// Return item at index -inline -DeltaInfo* DIV_get(const DeltaInfoVector* vec, Py_ssize_t i) -{ - assert(i < vec->size && vec->mem); - return &vec->mem[i]; -} - -// Return last item -inline -DeltaInfo* DIV_last(const DeltaInfoVector* vec) -{ - return DIV_get(vec, vec->size-1); -} - -inline -int DIV_empty(const DeltaInfoVector* vec) -{ - return vec->size == 0; -} - -// Return end pointer of the vector -inline -const DeltaInfo* DIV_end(const DeltaInfoVector* vec) -{ - assert(!DIV_empty(vec)); - return vec->mem + vec->size; -} - -// return first item in vector -inline -DeltaInfo* DIV_first(const DeltaInfoVector* vec) -{ - assert(!DIV_empty(vec)); - return vec->mem; -} - -// return rbound offset in bytes. We use information contained in the -// vec to do that -inline -uint DIV_info_rbound(const DeltaInfoVector* vec, const DeltaInfo* di) -{ - if (DIV_last(vec) == di){ - return di->to + vec->di_last_size; - } else { - return (di+1)->to; - } -} - -// return size of the given delta info item -inline -uint DIV_info_size2(const DeltaInfoVector* vec, const DeltaInfo* di, const DeltaInfo* const veclast) -{ - if (veclast == di){ - return vec->di_last_size; - } else { - return (di+1)->to - di->to; - } -} - -// return size of the given delta info item -inline -uint DIV_info_size(const DeltaInfoVector* vec, const DeltaInfo* di) -{ - return DIV_info_size2(vec, di, DIV_last(vec)); -} - -void DIV_destroy(DeltaInfoVector* vec) -{ - if (vec->mem){ -#ifdef DEBUG - fprintf(stderr, "DIV_destroy: %p\n", (void*)vec->mem); -#endif - PyMem_Free(vec->mem); - vec->size = 0; - vec->reserved_size = 0; - vec->mem = 0; - } -} - -// Reset this vector so that its existing memory can be filled again. -// Memory will be kept, but not cleaned up -inline -void DIV_forget_members(DeltaInfoVector* vec) -{ - vec->size = 0; -} - -// Reset the vector so that its size will be zero -// It will keep its memory though, and hence can be filled again -inline -void DIV_reset(DeltaInfoVector* vec) -{ - if (vec->size == 0) - return; - vec->size = 0; -} - - -// Append one chunk to the end of the list, and return a pointer to it -// It will not have been initialized ! -inline -DeltaInfo* DIV_append(DeltaInfoVector* vec) -{ - if (vec->size + 1 > vec->reserved_size){ - DIV_grow_by(vec, gDIV_grow_by); - } - - DeltaInfo* next = vec->mem + vec->size; - vec->size += 1; - return next; -} - -// Return delta chunk being closest to the given absolute offset -inline -DeltaInfo* DIV_closest_chunk(const DeltaInfoVector* vec, ull ofs) -{ - assert(vec->mem); - - ull lo = 0; - ull hi = vec->size; - ull mid; - DeltaInfo* di; - - while (lo < hi) - { - mid = (lo + hi) / 2; - di = vec->mem + mid; - if (di->to > ofs){ - hi = mid; - } else if ((DIV_info_rbound(vec, di) > ofs) | (di->to == ofs)) { - return di; - } else { - lo = mid + 1; - } - } - - return DIV_last(vec); -} - - -// Return the amount of chunks a slice at the given spot would have, as well as -// its size in bytes it would have if the possibly partial chunks would be encoded -// and added to the spot marked by sdc -uint DIV_count_slice_bytes(const DeltaInfoVector* src, uint ofs, uint size) -{ - uint num_bytes = 0; - DeltaInfo* cdi = DIV_closest_chunk(src, ofs); - - DeltaChunk dc; - DC_init(&dc, 0, 0, 0, NULL); - - // partial overlap - if (cdi->to != ofs) { - const ull relofs = ofs - cdi->to; - const uint cdisize = DIV_info_size(src, cdi); - const uint max_size = cdisize - relofs < size ? cdisize - relofs : size; - size -= max_size; - - // get the size in bytes the info would have - next_delta_info(src->dstream + cdi->dso, &dc); - dc.so += relofs; - dc.ts = max_size; - num_bytes += DC_count_encode_bytes(&dc); - - cdi += 1; - - if (size == 0){ - return num_bytes; - } - } - - const DeltaInfo* const vecend = DIV_end(src); - const uchar* nstream; - for( ;cdi < vecend; ++cdi){ - nstream = next_delta_info(src->dstream + cdi->dso, &dc); - - if (dc.ts < size) { - num_bytes += nstream - (src->dstream + cdi->dso); - size -= dc.ts; - } else { - dc.ts = size; - num_bytes += DC_count_encode_bytes(&dc); - size = 0; - break; - } - } - - assert(size == 0); - return num_bytes; -} - -// Write a slice as defined by its absolute offset in bytes and its size into the given -// destination memory. The individual chunks written will be a byte copy of the source -// data chunk stream -// Return: number of chunks in the slice -uint DIV_copy_slice_to(const DeltaInfoVector* src, uchar** dest, ull tofs, uint size) -{ - assert(DIV_lbound(src) <= tofs); - assert((tofs + size) <= DIV_info_rbound(src, DIV_last(src))); - - DeltaChunk dc; - DC_init(&dc, 0, 0, 0, NULL); - - DeltaInfo* cdi = DIV_closest_chunk(src, tofs); - uint num_chunks = 0; - - // partial overlap - if (cdi->to != tofs) { - const uint relofs = tofs - cdi->to; - next_delta_info(src->dstream + cdi->dso, &dc); - const uint max_size = dc.ts - relofs < size ? dc.ts - relofs : size; - - size -= max_size; - - // adjust dc proportions - DC_encode_to(&dc, dest, relofs, max_size); - - num_chunks += 1; - cdi += 1; - - if (size == 0){ - return num_chunks; - } - } - - const uchar* dstream = src->dstream + cdi->dso; - const uchar* nstream = dstream; - for( ; nstream; dstream = nstream) - { - num_chunks += 1; - nstream = next_delta_info(dstream, &dc); - if (dc.ts < size) { - memcpy(*dest, dstream, nstream - dstream); - *dest += nstream - dstream; - size -= dc.ts; - } else { - DC_encode_to(&dc, dest, 0, size); - size = 0; - break; - } - } - - assert(size == 0); - return num_chunks; -} - - -// Take slices of div into the corresponding area of the tsi, which is the topmost -// delta to apply. -bool DIV_connect_with_base(ToplevelStreamInfo* tsi, DeltaInfoVector* div) -{ - assert(tsi->num_chunks); - - - uint num_bytes = 0; - const uchar* data = TSI_first(tsi); - const uchar* dend = TSI_end(tsi); - - DeltaChunk dc; - DC_init(&dc, 0, 0, 0, NULL); - - - // COMPUTE SIZE OF TARGET STREAM - ///////////////////////////////// - for (;data < dend;) - { - data = next_delta_info(data, &dc); - - // Data chunks don't need processing - if (dc.data){ - num_bytes += 1 + dc.ts; - continue; - } - - num_bytes += DIV_count_slice_bytes(div, dc.so, dc.ts); - } - assert(DC_rbound(&dc) == tsi->target_size); - - - // GET NEW DELTA BUFFER - //////////////////////// - uchar *const dstream = PyMem_Malloc(num_bytes); - if (!dstream){ - return 0; - } - - - data = TSI_first(tsi); - const uchar *ndata = data; - dend = TSI_end(tsi); - - uint num_chunks = 0; - uchar* ds = dstream; - DC_init(&dc, 0, 0, 0, NULL); - - // pick slices from the delta and put them into the new stream - for (; data < dend; data = ndata) - { - ndata = next_delta_info(data, &dc); - - // Data chunks don't need processing - if (dc.data){ - // just copy it over - memcpy((void*)ds, (void*)data, ndata - data); - ds += ndata - data; - num_chunks += 1; - continue; - } - - // Copy Chunks - num_chunks += DIV_copy_slice_to(div, &ds, dc.so, dc.ts); - } - assert(ds - dstream == num_bytes); - assert(num_chunks >= tsi->num_chunks); - assert(DC_rbound(&dc) == tsi->target_size); - - // finally, replace the streams - TSI_replace_stream(tsi, dstream, num_bytes); - tsi->cstart = dstream; // we have NO header ! - assert(tsi->tds == dstream); - tsi->num_chunks = num_chunks; - - - return 1; - -} - -// DELTA CHUNK LIST (PYTHON) -///////////////////////////// -// Internally, it has nothing to do with a ChunkList anymore though -typedef struct { - PyObject_HEAD - // ----------- - ToplevelStreamInfo istream; - -} DeltaChunkList; - - - -int DCL_init(DeltaChunkList*self, PyObject *args, PyObject *kwds) -{ - if(args && PySequence_Size(args) > 0){ - PyErr_SetString(PyExc_ValueError, "Too many arguments"); - return -1; - } - - TSI_init(&self->istream); - return 0; -} - - -void DCL_dealloc(DeltaChunkList* self) -{ - TSI_destroy(&(self->istream)); -} - - -PyObject* DCL_py_rbound(DeltaChunkList* self) -{ - return PyLong_FromUnsignedLongLong(self->istream.target_size); -} - -// Write using a write function, taking remaining bytes from a base buffer - -PyObject* DCL_apply(DeltaChunkList* self, PyObject* args) -{ - PyObject* pybuf = 0; - PyObject* writeproc = 0; - if (!PyArg_ParseTuple(args, "OO", &pybuf, &writeproc)){ - PyErr_BadArgument(); - return NULL; - } - - if (!PyObject_CheckReadBuffer(pybuf)){ - PyErr_SetString(PyExc_ValueError, "First argument must be a buffer-compatible object, like a string, or a memory map"); - return NULL; - } - - if (!PyCallable_Check(writeproc)){ - PyErr_SetString(PyExc_ValueError, "Second argument must be a writer method with signature write(buf)"); - return NULL; - } - - const uchar* base; - Py_ssize_t baselen; - PyObject_AsReadBuffer(pybuf, (const void**)&base, &baselen); - - PyObject* tmpargs = PyTuple_New(1); - - const uchar* data = TSI_first(&self->istream); - const uchar* const dend = TSI_end(&self->istream); - - DeltaChunk dc; - DC_init(&dc, 0, 0, 0, NULL); - - while (data < dend){ - data = next_delta_info(data, &dc); - DC_apply(&dc, base, writeproc, tmpargs); - } - - Py_DECREF(tmpargs); - Py_RETURN_NONE; -} - -PyMethodDef DCL_methods[] = { - {"apply", (PyCFunction)DCL_apply, METH_VARARGS, "Apply the given iterable of delta streams" }, - {"rbound", (PyCFunction)DCL_py_rbound, METH_NOARGS, NULL}, - {NULL} /* Sentinel */ -}; - -PyTypeObject DeltaChunkListType = { - PyObject_HEAD_INIT(NULL) - 0, /*ob_size*/ - "DeltaChunkList", /*tp_name*/ - sizeof(DeltaChunkList), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)DCL_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - "Minimal Delta Chunk List",/* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - DCL_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)DCL_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; - - -// Makes a new copy of the DeltaChunkList - you have to do everything yourselve -// in C ... want C++ !! -DeltaChunkList* DCL_new_instance(void) -{ - DeltaChunkList* dcl = (DeltaChunkList*) PyType_GenericNew(&DeltaChunkListType, 0, 0); - assert(dcl); - - DCL_init(dcl, 0, 0); - return dcl; -} - -// Read the next delta chunk from the given stream and advance it -// dc will contain the parsed information, its offset must be set by -// the previous call of next_delta_info, which implies it should remain the -// same instance between the calls. -// Return the altered uchar pointer, reassign it to the input data -inline -const uchar* next_delta_info(const uchar* data, DeltaChunk* dc) -{ - const char cmd = *data++; - - if (cmd & 0x80) - { - uint cp_off = 0, cp_size = 0; - if (cmd & 0x01) cp_off = *data++; - if (cmd & 0x02) cp_off |= (*data++ << 8); - if (cmd & 0x04) cp_off |= (*data++ << 16); - if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); - if (cmd & 0x10) cp_size = *data++; - if (cmd & 0x20) cp_size |= (*data++ << 8); - if (cmd & 0x40) cp_size |= (*data++ << 16); // this should never get hit with current deltas ... - if (cp_size == 0) cp_size = 0x10000; - - dc->to += dc->ts; - dc->data = NULL; - dc->so = cp_off; - dc->ts = cp_size; - - } else if (cmd) { - // Just share the data - dc->to += dc->ts; - dc->data = data; - dc->ts = cmd; - dc->so = 0; - - data += cmd; - } else { - PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); - assert(0); - return NULL; - } - - return data; -} - -// Return amount of chunks encoded in the given delta stream -// If read_header is True, then the header msb chunks will be read first. -// Otherwise, the stream is assumed to be scrubbed one past the header -uint compute_chunk_count(const uchar* data, const uchar* dend, bool read_header) -{ - // read header - if (read_header){ - msb_size(&data, dend); - msb_size(&data, dend); - } - - DeltaChunk dc; - DC_init(&dc, 0, 0, 0, NULL); - uint num_chunks = 0; - - while (data < dend) - { - data = next_delta_info(data, &dc); - num_chunks += 1; - }// END handle command opcodes - - return num_chunks; -} - -PyObject* connect_deltas(PyObject *self, PyObject *dstreams) -{ - // obtain iterator - PyObject* stream_iter = 0; - if (!PyIter_Check(dstreams)){ - stream_iter = PyObject_GetIter(dstreams); - if (!stream_iter){ - PyErr_SetString(PyExc_RuntimeError, "Couldn't obtain iterator for streams"); - return NULL; - } - } else { - stream_iter = dstreams; - } - - DeltaInfoVector div; - ToplevelStreamInfo tdsinfo; - TSI_init(&tdsinfo); - DIV_init(&div, 0); - - - // GET TOPLEVEL DELTA STREAM - int error = 0; - PyObject* ds = 0; - unsigned int dsi = 0; // delta stream index we process - ds = PyIter_Next(stream_iter); - if (!ds){ - error = 1; - goto _error; - } - - dsi += 1; - tdsinfo.parent_object = PyObject_CallMethod(ds, "read", 0); - if (!PyObject_CheckReadBuffer(tdsinfo.parent_object)){ - Py_DECREF(ds); - error = 1; - goto _error; - } - - PyObject_AsReadBuffer(tdsinfo.parent_object, (const void**)&tdsinfo.tds, &tdsinfo.tdslen); - if (tdsinfo.tdslen > pow(2, 32)){ - // parent object is deallocated by info structure - Py_DECREF(ds); - PyErr_SetString(PyExc_RuntimeError, "Cannot handle deltas larger than 4GB"); - tdsinfo.parent_object = 0; - - error = 1; - goto _error; - } - Py_DECREF(ds); - - // let it officially know, and initialize its internal state - TSI_set_stream(&tdsinfo, tdsinfo.tds); - - // INTEGRATE ANCESTOR DELTA STREAMS - for (ds = PyIter_Next(stream_iter); ds != NULL; ds = PyIter_Next(stream_iter), ++dsi) - { - // Its important to initialize this before the next block which can jump - // to code who needs this to exist ! - PyObject* db = 0; - - // When processing the first delta, we know we will have to alter the tds - // Hence we copy it and deallocate the parent object - if (dsi == 1) { - if (!TSI_copy_stream_from_object(&tdsinfo)){ - PyErr_SetString(PyExc_RuntimeError, "Could not allocate memory to copy toplevel buffer"); - // info structure takes care of the parent_object - error = 1; - goto loop_end; - } - - tdsinfo.num_chunks = compute_chunk_count(tdsinfo.cstart, TSI_end(&tdsinfo), 0); - } - - db = PyObject_CallMethod(ds, "read", 0); - if (!PyObject_CheckReadBuffer(db)){ - error = 1; - PyErr_SetString(PyExc_RuntimeError, "Returned buffer didn't support the buffer protocol"); - goto loop_end; - } - - // Fill the stream info structure - const uchar* data; - Py_ssize_t dlen; - PyObject_AsReadBuffer(db, (const void**)&data, &dlen); - const uchar* const dstart = data; - const uchar* const dend = data + dlen; - div.dstream = dstart; - - if (dlen > pow(2, 32)){ - error = 1; - PyErr_SetString(PyExc_RuntimeError, "Cannot currently handle deltas larger than 4GB"); - goto loop_end; - } - - // READ HEADER - msb_size(&data, dend); - const ull target_size = msb_size(&data, dend); - - DIV_reserve_memory(&div, compute_chunk_count(data, dend, 0)); - - // parse command stream - DeltaInfo* di = 0; // temporary pointer - DeltaChunk dc; - DC_init(&dc, 0, 0, 0, NULL); - - assert(data < dend); - while (data < dend) - { - di = DIV_append(&div); - di->dso = data - dstart; - if ((data = next_delta_info(data, &dc))){ - di->to = dc.to; - } else { - error = 1; - goto loop_end; - } - }// END handle command opcodes - - // finalize information - div.di_last_size = dc.ts; - - if (DC_rbound(&dc) != target_size){ - PyErr_SetString(PyExc_RuntimeError, "Failed to parse delta stream"); - error = 1; - } - - #ifdef DEBUG - fprintf(stderr, "------------ Stream %i --------\n ", (int)dsi); - fprintf(stderr, "Before Connect: tdsinfo: num_chunks = %i, bytelen = %i KiB, target_size = %i KiB\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen/1000, (int)tdsinfo.target_size/1000); - fprintf(stderr, "div->num_chunks = %i, div->reserved_size = %i, div->bytelen=%i KiB\n", (int)div.size, (int)div.reserved_size, (int)dlen/1000); - #endif - - if (!DIV_connect_with_base(&tdsinfo, &div)){ - error = 1; - } - - #ifdef DEBUG - fprintf(stderr, "after connect: tdsinfo->num_chunks = %i, tdsinfo->bytelen = %i KiB\n", (int)tdsinfo.num_chunks, (int)tdsinfo.tdslen/1000); - #endif - - // destroy members, but keep memory - DIV_reset(&div); - -loop_end: - // perform cleanup - Py_DECREF(ds); - Py_DECREF(db); - - if (error){ - break; - } - }// END for each stream object - - if (dsi == 0){ - PyErr_SetString(PyExc_ValueError, "No streams provided"); - } - - -_error: - - if (stream_iter != dstreams){ - Py_DECREF(stream_iter); - } - - - DIV_destroy(&div); - - // Return the actual python object - its just a container - DeltaChunkList* dcl = DCL_new_instance(); - if (!dcl){ - PyErr_SetString(PyExc_RuntimeError, "Couldn't allocate list"); - // Otherwise tdsinfo would be deallocated by the chunk list - TSI_destroy(&tdsinfo); - error = 1; - } else { - // Plain copy, transfer ownership to dcl - dcl->istream = tdsinfo; - } - - if (error){ - // Will dealloc tdcv - Py_XDECREF(dcl); - return NULL; - } - - return (PyObject*)dcl; -} - - -// Write using a write function, taking remaining bytes from a base buffer -// replaces the corresponding method in python -PyObject* apply_delta(PyObject* self, PyObject* args) -{ - PyObject* pybbuf = 0; - PyObject* pydbuf = 0; - PyObject* pytbuf = 0; - if (!PyArg_ParseTuple(args, "OOO", &pybbuf, &pydbuf, &pytbuf)){ - PyErr_BadArgument(); - return NULL; - } - - PyObject* objects[] = { pybbuf, pydbuf, pytbuf }; - assert(sizeof(objects) / sizeof(PyObject*) == 3); - - uint i; - for(i = 0; i < 3; i++){ - if (!PyObject_CheckReadBuffer(objects[i])){ - PyErr_SetString(PyExc_ValueError, "Argument must be a buffer-compatible object, like a string, or a memory map"); - return NULL; - } - } - - Py_ssize_t lbbuf; Py_ssize_t ldbuf; Py_ssize_t ltbuf; - const uchar* bbuf; const uchar* dbuf; - uchar* tbuf; - PyObject_AsReadBuffer(pybbuf, (const void**)(&bbuf), &lbbuf); - PyObject_AsReadBuffer(pydbuf, (const void**)(&dbuf), &ldbuf); - - if (PyObject_AsWriteBuffer(pytbuf, (void**)(&tbuf), <buf)){ - PyErr_SetString(PyExc_ValueError, "Argument 3 must be a writable buffer"); - return NULL; - } - - const uchar* data = dbuf; - const uchar* dend = dbuf + ldbuf; - - while (data < dend) - { - const char cmd = *data++; - - if (cmd & 0x80) - { - unsigned long cp_off = 0, cp_size = 0; - if (cmd & 0x01) cp_off = *data++; - if (cmd & 0x02) cp_off |= (*data++ << 8); - if (cmd & 0x04) cp_off |= (*data++ << 16); - if (cmd & 0x08) cp_off |= ((unsigned) *data++ << 24); - if (cmd & 0x10) cp_size = *data++; - if (cmd & 0x20) cp_size |= (*data++ << 8); - if (cmd & 0x40) cp_size |= (*data++ << 16); - if (cp_size == 0) cp_size = 0x10000; - - memcpy(tbuf, bbuf + cp_off, cp_size); - tbuf += cp_size; - - } else if (cmd) { - memcpy(tbuf, data, cmd); - tbuf += cmd; - data += cmd; - } else { - PyErr_SetString(PyExc_RuntimeError, "Encountered an unsupported delta cmd: 0"); - return NULL; - } - }// END handle command opcodes - - Py_RETURN_NONE; -} diff --git a/gitdb/_delta_apply.h b/gitdb/_delta_apply.h deleted file mode 100644 index 1fcd53832..000000000 --- a/gitdb/_delta_apply.h +++ /dev/null @@ -1,6 +0,0 @@ -#include - -extern PyObject* connect_deltas(PyObject *self, PyObject *dstreams); -extern PyObject* apply_delta(PyObject* self, PyObject* args); - -extern PyTypeObject DeltaChunkListType; diff --git a/gitdb/_fun.c b/gitdb/_fun.c deleted file mode 100644 index 49970386f..000000000 --- a/gitdb/_fun.c +++ /dev/null @@ -1,107 +0,0 @@ -#include -#include "_delta_apply.h" - -static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args) -{ - const unsigned char *sha; - const unsigned int sha_len; - - // Note: self is only set if we are a c type. We emulate an instance method, - // hence we have to get the instance as 'first' argument - - // get instance and sha - PyObject* inst = 0; - if (!PyArg_ParseTuple(args, "Os#", &inst, &sha, &sha_len)) - return NULL; - - if (sha_len != 20) { - PyErr_SetString(PyExc_ValueError, "Sha is not 20 bytes long"); - return NULL; - } - - if( !inst){ - PyErr_SetString(PyExc_ValueError, "Cannot be called without self"); - return NULL; - } - - // read lo and hi bounds - PyObject* fanout_table = PyObject_GetAttrString(inst, "_fanout_table"); - if (!fanout_table){ - PyErr_SetString(PyExc_ValueError, "Couldn't obtain fanout table"); - return NULL; - } - - unsigned int lo = 0, hi = 0; - if (sha[0]){ - PyObject* item = PySequence_GetItem(fanout_table, (const Py_ssize_t)(sha[0]-1)); - lo = PyInt_AS_LONG(item); - Py_DECREF(item); - } - PyObject* item = PySequence_GetItem(fanout_table, (const Py_ssize_t)sha[0]); - hi = PyInt_AS_LONG(item); - Py_DECREF(item); - item = 0; - - Py_DECREF(fanout_table); - - // get sha query function - PyObject* get_sha = PyObject_GetAttrString(inst, "sha"); - if (!get_sha){ - PyErr_SetString(PyExc_ValueError, "Couldn't obtain sha method"); - return NULL; - } - - PyObject *sha_str = 0; - while (lo < hi) { - const int mid = (lo + hi)/2; - sha_str = PyObject_CallFunction(get_sha, "i", mid); - if (!sha_str) { - return NULL; - } - - // we really trust that string ... for speed - const int cmp = memcmp(PyString_AS_STRING(sha_str), sha, 20); - Py_DECREF(sha_str); - sha_str = 0; - - if (cmp < 0){ - lo = mid + 1; - } - else if (cmp > 0) { - hi = mid; - } - else { - Py_DECREF(get_sha); - return PyInt_FromLong(mid); - }// END handle comparison - }// END while lo < hi - - // nothing found, cleanup - Py_DECREF(get_sha); - Py_RETURN_NONE; -} - -static PyMethodDef py_fun[] = { - { "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, "TODO" }, - { "connect_deltas", (PyCFunction)connect_deltas, METH_O, "See python implementation" }, - { "apply_delta", (PyCFunction)apply_delta, METH_VARARGS, "See python implementation" }, - { NULL, NULL, 0, NULL } -}; - -#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ -#define PyMODINIT_FUNC void -#endif -PyMODINIT_FUNC init_perf(void) -{ - PyObject *m; - - if (PyType_Ready(&DeltaChunkListType) < 0) - return; - - m = Py_InitModule3("_perf", py_fun, NULL); - if (m == NULL) - return; - - Py_INCREF(&DeltaChunkListType); - PyModule_AddObject(m, "DeltaChunkList", (PyObject *)&DeltaChunkListType); -} diff --git a/gitdb/fun.py b/gitdb/fun.py index ac9d99395..8ca38c867 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -776,6 +776,6 @@ def is_equal_canonical_sha(canonical_length, match, sha1): try: - from _perf import connect_deltas + from gitdb_speedups._perf import connect_deltas except ImportError: pass diff --git a/gitdb/pack.py b/gitdb/pack.py index 2447455b5..20a451549 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -35,7 +35,7 @@ ) try: - from _perf import PackIndexFile_sha_to_index + from gitdb_speedups._perf import PackIndexFile_sha_to_index except ImportError: pass # END try c module diff --git a/gitdb/stream.py b/gitdb/stream.py index be95c113a..2f4c12dfb 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -33,7 +33,7 @@ has_perf_mod = False PY26 = sys.version_info[:2] < (2, 7) try: - from _perf import apply_delta as c_apply_delta + from gitdb_speedups._perf import apply_delta as c_apply_delta has_perf_mod = True except ImportError: pass diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..3c6e79cf3 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py index c2f9f1eb1..eccb95278 100755 --- a/setup.py +++ b/setup.py @@ -1,125 +1,45 @@ -#!/usr/bin/env python -from distutils.core import setup, Extension -from distutils.command.build_py import build_py -from distutils.command.build_ext import build_ext +from setuptools import setup -import os -import sys +# NOTE: This is currently duplicated from the gitdb.__init__ module, because +# that's just how you write a setup.py (nobody reads this stuff out of the +# module) -# wow, this is a mixed bag ... I am pretty upset about all of this ... -setuptools_build_py_module = None -try: - # don't pull it in if we don't have to - if 'setuptools' in sys.modules: - import setuptools.command.build_py as setuptools_build_py_module - from setuptools.command.build_ext import build_ext -except ImportError: - pass - - -class build_ext_nofail(build_ext): - - """Doesn't fail when build our optional extensions""" - - def run(self): - try: - build_ext.run(self) - except Exception: - print("Ignored failure when building extensions, pure python modules will be used instead") - # END ignore errors - - -def get_data_files(self): - """Can you feel the pain ? So, in python2.5 and python2.4 coming with maya, - the line dealing with the ``plen`` has a bug which causes it to truncate too much. - It is fixed in the system interpreters as they receive patches, and shows how - bad it is if something doesn't have proper unittests. - The code here is a plain copy of the python2.6 version which works for all. - - Generate list of '(package,src_dir,build_dir,filenames)' tuples""" - data = [] - if not self.packages: - return data - - # this one is just for the setup tools ! They don't iniitlialize this variable - # when they should, but do it on demand using this method.Its crazy - if hasattr(self, 'analyze_manifest'): - self.analyze_manifest() - # END handle setuptools ... - - for package in self.packages: - # Locate package source directory - src_dir = self.get_package_dir(package) - - # Compute package build directory - build_dir = os.path.join(*([self.build_lib] + package.split('.'))) - - # Length of path to strip from found files - plen = 0 - if src_dir: - plen = len(src_dir) + 1 - - # Strip directory from globbed filenames - filenames = [ - file[plen:] for file in self.find_data_files(package, src_dir) - ] - data.append((package, src_dir, build_dir, filenames)) - return data - -build_py.get_data_files = get_data_files -if setuptools_build_py_module: - setuptools_build_py_module.build_py._get_data_files = get_data_files -# END apply setuptools patch too - -# NOTE: This is currently duplicated from the gitdb.__init__ module, as we cannot -# satisfy the dependencies at installation time, unfortunately, due to inherent limitations -# of distutils, which cannot install the prerequesites of a package before the acutal package. __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" version_info = (0, 6, 4) __version__ = '.'.join(str(i) for i in version_info) -setup(cmdclass={'build_ext': build_ext_nofail}, - name="gitdb", - version=__version__, - description="Git Object Database", - author=__author__, - author_email=__contact__, - url=__homepage__, - packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), - package_dir = {'gitdb': 'gitdb'}, - ext_modules=[Extension('gitdb._perf', ['gitdb/_fun.c', 'gitdb/_delta_apply.c'], include_dirs=['gitdb'])], - license = "BSD License", - zip_safe=False, - requires=('smmap (>=0.8.5)', ), - install_requires=('smmap >= 0.8.5'), - long_description = """GitDB is a pure-Python git object database""", - # See https://pypi.python.org/pypi?%3Aaction=list_classifiers - classifiers=[ - # Picked from - # http://pypi.python.org/pypi?:action=list_classifiers - #"Development Status :: 1 - Planning", - #"Development Status :: 2 - Pre-Alpha", - #"Development Status :: 3 - Alpha", - # "Development Status :: 4 - Beta", - "Development Status :: 5 - Production/Stable", - #"Development Status :: 6 - Mature", - #"Development Status :: 7 - Inactive", - "Environment :: Console", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Operating System :: POSIX", - "Operating System :: Microsoft :: Windows", - "Operating System :: MacOS :: MacOS X", - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", -],) +setup( + name="gitdb", + version=__version__, + description="Git Object Database", + author=__author__, + author_email=__contact__, + url=__homepage__, + packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), + license="BSD License", + zip_safe=False, + install_requires=['smmap >= 0.8.5'], + long_description="""GitDB is a pure-Python git object database""", + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + ] +) From 55aa3d79607fdf9123d266612d58a65e04a8622e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 9 Oct 2016 10:31:02 +0200 Subject: [PATCH 0327/1392] doc(README): correct appveyor badge And remove some broken badges. --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index ef0b23601..83c674456 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Although memory maps have many advantages, they represent a very limited system ## Overview [![Build Status](https://travis-ci.org/gitpython-developers/smmap.svg?branch=master)](https://travis-ci.org/gitpython-developers/smmap) -[![Build status](https://ci.appveyor.com/api/projects/status/h8rl7thsr42oc0pf?svg=true&passingText=windows%20OK&failingText=windows%20failed)](https://ci.appveyor.com/project/Byron/smmap) +[![Build status](https://ci.appveyor.com/api/projects/status/kuws846av5lvmugo?svg=true&passingText=windows%20OK&failingText=windows%20failed)](https://ci.appveyor.com/project/Byron/smmap) [![Coverage Status](https://coveralls.io/repos/gitpython-developers/smmap/badge.png)](https://coveralls.io/r/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/pr)](http://www.issuestats.com/github/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/issue)](http://www.issuestats.com/github/gitpython-developers/smmap) @@ -40,8 +40,6 @@ The package was tested on all of the previously mentioned configurations. ## Installing smmap -[![Latest Version](https://pypip.in/version/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) -[![Supported Python versions](https://pypip.in/py_versions/smmap/badge.svg)](https://pypi.python.org/pypi/smmap/) [![Documentation Status](https://readthedocs.org/projects/smmap/badge/?version=latest)](https://readthedocs.org/projects/smmap/?badge=latest) Its easiest to install smmap using the [pip](http://www.pip-installer.org/en/latest) program: From 09023b1013e081755dca575f62c35eb93ee8065a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 16 Oct 2016 12:10:19 +0200 Subject: [PATCH 0328/1392] chore(rename): smmap -> smmap2 [skip ci] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f6a04827d..2c519e648 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ long_description = "See http://github.com/gitpython-developers/smmap" setup( - name="smmap", + name="smmap2", version=smmap.__version__, description="A pure python implementation of a sliding window memory map manager", author=smmap.__author__, From ac5df7061ee11232346b3d0eb3aa5b43eebc847d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 16 Oct 2016 12:18:20 +0200 Subject: [PATCH 0329/1392] chore(version): set version to 2.0 Just to match the name a bit better. [skip ci] --- smmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index e711bbbf3..8b4059c87 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (0, 9, 0) +version_info = (2, 0, 0) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 38866bc7c4956170c681a62c4508f934ac826469 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 16 Oct 2016 12:23:27 +0200 Subject: [PATCH 0330/1392] chore(rename): gitdb2 v2.0 v2 is chosen to better match the name. --- gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 6554cf904..bfa083bb3 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 4) +version_info = (2, 0, 0) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 18e4aea23..ac5df7061 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 18e4aea23644ea43657cb2e6846b6aaf78720c27 +Subproject commit ac5df7061ee11232346b3d0eb3aa5b43eebc847d diff --git a/setup.py b/setup.py index eccb95278..f7e3761ab 100755 --- a/setup.py +++ b/setup.py @@ -7,11 +7,11 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (0, 6, 4) +version_info = (2, 0, 0) __version__ = '.'.join(str(i) for i in version_info) setup( - name="gitdb", + name="gitdb2", version=__version__, description="Git Object Database", author=__author__, @@ -20,7 +20,7 @@ packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), license="BSD License", zip_safe=False, - install_requires=['smmap >= 0.8.5'], + install_requires=['smmap2 >= 2.0.0'], long_description="""GitDB is a pure-Python git object database""", # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ From 6a217abbbf1673ab2e5794a3cc0bc151e16b9bc0 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 1 Oct 2016 22:28:40 +0200 Subject: [PATCH 0331/1392] ci: Test on Appveyor for Windows. --- .appveyor.yml | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000..2daadaa4d --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,49 @@ +# CI on Windows via appveyor +environment: + + matrix: + ## MINGW + # + - PYTHON: "C:\\Python27" + PYTHON_VERSION: "2.7" + - PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4" + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5" + - PYTHON: "C:\\Miniconda35-x64" + PYTHON_VERSION: "3.5" + IS_CONDA: "yes" + +install: + - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% + + ## Print configuration for debugging. + # + - | + echo %PATH% + uname -a + where python pip pip2 pip3 pip34 + python --version + python -c "import struct; print(struct.calcsize('P') * 8)" + + - IF "%IS_CONDA%"=="yes" ( + conda info -a & + conda install --yes --quiet pip + ) + - pip install nose wheel coveralls + + ## For commits performed with the default user. + - | + git config --global user.email "travis@ci.com" + git config --global user.name "Travis Runner" + + - pip install -e . + +build: false + +test_script: + - IF "%PYTHON_VERSION%"=="3.5" ( + nosetests -v --with-coverage + ) ELSE ( + nosetests -v + ) From 62202bbbb58379814c44bd26cf662e68d3fa6dbb Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 1 Oct 2016 22:34:51 +0200 Subject: [PATCH 0332/1392] appveyor: Add badge on ankostis repo for testing. --- README.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 1b754d09b..ca51dfaba 100644 --- a/README.rst +++ b/README.rst @@ -43,8 +43,8 @@ Once the clone is complete, please be sure to initialize the submodules using cd gitdb git submodule update --init -Run the tests with - +Run the tests with + nosetests DEVELOPMENT @@ -52,13 +52,12 @@ DEVELOPMENT .. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master :target: https://travis-ci.org/gitpython-developers/gitdb - +.. image:: https://ci.appveyor.com/api/projects/status/2qa4km4ln7bfv76r/branch/master?svg=true&passingText=windows%20OK&failingText=windows%20failed + :target: https://ci.appveyor.com/project/ankostis/gitpython/branch/master) .. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png :target: https://coveralls.io/r/gitpython-developers/gitdb - .. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/pr :target: http://www.issuestats.com/github/gitpython-developers/gitdb - .. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/issue :target: http://www.issuestats.com/github/gitpython-developers/gitdb From 587dc4ec1311e135d70996a077a2f978e303d3fc Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 1 Oct 2016 23:08:31 +0200 Subject: [PATCH 0333/1392] TCs: fix div-by-zero on elapsed times (appveyor CPU is fast!) --- gitdb/test/performance/test_pack.py | 10 +++++----- gitdb/test/performance/test_pack_streaming.py | 6 +++--- gitdb/test/performance/test_stream.py | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index bdd2b0a37..fc8d9d54f 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -36,7 +36,7 @@ def test_pack_random_access(self): sha_list = list(pdb.sha_iter()) elapsed = time() - st ns = len(sha_list) - print("PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed), file=sys.stderr) + print("PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / (elapsed or 1)), file=sys.stderr) # sha lookup: best-case and worst case access pdb_pack_info = pdb._pack_info @@ -51,7 +51,7 @@ def test_pack_random_access(self): del(pdb._entities) pdb.entities() print("PDB: looked up %i sha in %i packs in %f s ( %f shas/s )" % - (ns, len(pdb.entities()), elapsed, ns / elapsed), file=sys.stderr) + (ns, len(pdb.entities()), elapsed, ns / (elapsed or 1)), file=sys.stderr) # END for each random mode # query info and streams only @@ -62,7 +62,7 @@ def test_pack_random_access(self): pdb_fun(sha) elapsed = time() - st print("PDB: Obtained %i object %s by sha in %f s ( %f items/s )" % - (max_items, pdb_fun.__name__.upper(), elapsed, max_items / elapsed), file=sys.stderr) + (max_items, pdb_fun.__name__.upper(), elapsed, max_items / (elapsed or 1)), file=sys.stderr) # END for each function # retrieve stream and read all @@ -78,7 +78,7 @@ def test_pack_random_access(self): elapsed = time() - st total_kib = total_size / 1000 print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % - (max_items, total_kib, total_kib / elapsed, elapsed, max_items / elapsed), file=sys.stderr) + (max_items, total_kib, total_kib / (elapsed or 1), elapsed, max_items / (elapsed or 1)), file=sys.stderr) @skip_on_travis_ci def test_loose_correctness(self): @@ -129,5 +129,5 @@ def test_correctness(self): # END for each entity elapsed = time() - st print("PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % - (count, crc, elapsed, count / elapsed), file=sys.stderr) + (count, crc, elapsed, count / (elapsed or 1)), file=sys.stderr) # END for each verify mode diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index f805e59fa..76f0f4a07 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -52,14 +52,14 @@ def test_pack_writing(self): # END gather objects for pack-writing elapsed = time() - st print("PDB Streaming: Got %i streams by sha in in %f s ( %f streams/s )" % - (ni, elapsed, ni / elapsed), file=sys.stderr) + (ni, elapsed, ni / (elapsed or 1)), file=sys.stderr) st = time() PackEntity.write_pack((pdb.stream(sha) for sha in pdb.sha_iter()), ostream.write, object_count=ni) elapsed = time() - st total_kb = ostream.bytes_written() / 1000 print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % - (total_kb, elapsed, total_kb / elapsed), sys.stderr) + (total_kb, elapsed, total_kb / (elapsed or 1)), sys.stderr) @skip_on_travis_ci def test_stream_reading(self): @@ -82,4 +82,4 @@ def test_stream_reading(self): elapsed = time() - st total_kib = total_size / 1000 print(sys.stderr, "PDB Streaming: Got %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % - (ni, total_kib, total_kib / elapsed, elapsed, ni / elapsed), sys.stderr) + (ni, total_kib, total_kib / (elapsed or 1), elapsed, ni / (elapsed or 1)), sys.stderr) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index bd66b26ad..704f4d094 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -70,7 +70,7 @@ def test_large_data_streaming(self, path): size_kib = size / 1000 print("Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)" % - (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add), file=sys.stderr) + (size_kib, fsize_kib, desc, elapsed_add, size_kib / (elapsed_add or 1)), file=sys.stderr) # reading all at once st = time() @@ -81,7 +81,7 @@ def test_large_data_streaming(self, path): stream.seek(0) assert shadata == stream.getvalue() print("Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)" % - (size_kib, desc, elapsed_readall, size_kib / elapsed_readall), file=sys.stderr) + (size_kib, desc, elapsed_readall, size_kib / (elapsed_readall or 1)), file=sys.stderr) # reading in chunks of 1 MiB cs = 512 * 1000 @@ -101,7 +101,7 @@ def test_large_data_streaming(self, path): cs_kib = cs / 1000 print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" % - (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks), file=sys.stderr) + (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / (elapsed_readchunks or 1)), file=sys.stderr) # del db file so we keep something to do os.remove(db_file) From d48679a0b15feae754ebe9ef4c9d5809db0c0d08 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sun, 2 Oct 2016 01:10:47 +0200 Subject: [PATCH 0334/1392] tc: HALF FIX of `test_pack_entity ()` + On Windows, you cannot write onto a file held by another live file-pointer (test_pack.py:#L204). + The TC fails later, on clean up (the usual). --- gitdb/test/test_pack.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 601c0eaba..6e31363bf 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -188,7 +188,8 @@ def test_pack_entity(self, rw_dir): # pack writing - write all packs into one # index path can be None - pack_path = tempfile.mktemp('', "pack", rw_dir) + pack_path1 = tempfile.mktemp('', "pack1", rw_dir) + pack_path2 = tempfile.mktemp('', "pack2", rw_dir) index_path = tempfile.mktemp('', 'index', rw_dir) iteration = 0 @@ -196,7 +197,9 @@ def rewind_streams(): for obj in pack_objs: obj.stream.seek(0) # END utility - for ppath, ipath, num_obj in zip((pack_path, ) * 2, (index_path, None), (len(pack_objs), None)): + for ppath, ipath, num_obj in zip((pack_path1, pack_path2), + (index_path, None), + (len(pack_objs), None)): iwrite = None if ipath: ifile = open(ipath, 'wb') @@ -214,7 +217,7 @@ def rewind_streams(): assert os.path.getsize(ppath) > 100 # verify pack - pf = PackFile(ppath) + pf = PackFile(ppath) # FIXME: Leaks file-pointer(s)! assert pf.size() == len(pack_objs) assert pf.version() == PackFile.pack_version_default assert pf.checksum() == pack_sha From b2eafcc9c99c8b00a4adc2a316e800a9a7dc7319 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 22 Oct 2016 13:02:49 +0200 Subject: [PATCH 0335/1392] fix(MapWindow): unicode foes in read_into_memory() used by gitpython TCs Drop Windows only codepath bypassing memory-mapping due to some leaks in the past. Now Appveyor proves everything run ok. Additionally, this codepath had unicode problems on PY3. So deleting it, fixes 2 TCs in gitpython: + TestRepo.test_file_handle_leaks() + TestObjDbPerformance.test_random_access() See https://github.com/gitpython-developers/GitPython/issues/525 --- smmap/mman.py | 16 ++++++++++------ smmap/util.py | 25 +++---------------------- 2 files changed, 13 insertions(+), 28 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index af12ee98f..9df69ed29 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -31,9 +31,9 @@ class WindowCursor(object): __slots__ = ( '_manager', # the manger keeping all file regions '_rlist', # a regions list with regions for our file - '_region', # our current region or None - '_ofs', # relative offset from the actually mapped area to our start area - '_size' # maximum size we should provide + '_region', # our current class:`MapRegion` or None + '_ofs', # relative offset from the actually mapped area to our start area + '_size' # maximum size we should provide ) def __init__(self, manager=None, regions=None): @@ -308,10 +308,14 @@ def _collect_lru_region(self, size): if 0, we try to free any available region :return: Amount of freed regions - **Note:** We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. - If the system runs out of memory, it will tell. + .. Note:: + We don't raise exceptions anymore, in order to keep the system working, allowing temporary overallocation. + If the system runs out of memory, it will tell. - **todo:** implement a case where all unusued regions are discarded efficiently. Currently its only brute force""" + .. TODO:: + implement a case where all unusued regions are discarded efficiently. + Currently its only brute force + """ num_found = 0 while (size == 0) or (self._memory_size + size > self._max_memory_size): lru_region = None diff --git a/smmap/util.py b/smmap/util.py index 36ff1ab89..02df41a13 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -116,16 +116,13 @@ class MapRegion(object): '_size', # cached size of our memory map '__weakref__' ] - _need_compat_layer = sys.version_info[0] < 3 and sys.version_info[1] < 6 + _need_compat_layer = sys.version_info[:2] < (2, 6) if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset # END handle additional slot #{ Configuration - # Used for testing only. If True, all data will be loaded into memory at once. - # This makes sure no file handles will remain open. - _test_read_into_memory = False #} END configuration def __init__(self, path_or_fd, ofs, size, flags=0): @@ -160,10 +157,7 @@ def __init__(self, path_or_fd, ofs, size, flags=0): # bark that the size is too large ... many extra file accesses because # if this ... argh ! actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) - if self._test_read_into_memory: - self._mf = self._read_into_memory(fd, ofs, actual_size) - else: - self._mf = mmap(fd, actual_size, **kwargs) + self._mf = mmap(fd, actual_size, **kwargs) # END handle memory mode self._size = len(self._mf) @@ -179,19 +173,6 @@ def __init__(self, path_or_fd, ofs, size, flags=0): # We assume the first one to use us keeps us around self.increment_client_count() - def _read_into_memory(self, fd, offset, size): - """:return: string data as read from the given file descriptor, offset and size """ - os.lseek(fd, offset, os.SEEK_SET) - mf = '' - bytes_todo = size - while bytes_todo: - chunk = 1024 * 1024 - d = os.read(fd, chunk) - bytes_todo -= len(d) - mf += d - # END loop copy items - return mf - def __repr__(self): return "MapRegion<%i, %i>" % (self._b, self.size()) @@ -267,7 +248,7 @@ class MapRegionList(list): """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( '_path_or_fd', # path or file descriptor which is mapped by all our regions - '_file_size' # total size of the file we map + '_file_size' # total size of the file we map ) def __new__(cls, path): From 6e55a1c70843e5e5e29686176325f309f8285f29 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 22 Oct 2016 16:28:19 +0200 Subject: [PATCH 0336/1392] chore(version): v2.0.1 Better windows support --- smmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index 8b4059c87..9cfd0a1c1 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (2, 0, 0) +version_info = (2, 0, 1) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From a36fbeb41ce0e29d7f4f9eeb301c60032e47577c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 8 Dec 2016 11:58:31 +0100 Subject: [PATCH 0337/1392] doc(readme): make limitations way more prominent Also inform about the likelyhood of leaking system resources. [skip ci] --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 83c674456..247848d04 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,12 @@ When reading from many possibly large files in a fashion similar to random acces Although memory maps have many advantages, they represent a very limited system resource as every map uses one file descriptor, whose amount is limited per process. On 32 bit systems, the amount of memory you can have mapped at a time is naturally limited to theoretical 4GB of memory, which may not be enough for some applications. +## Limitations + +* **System resources (file-handles) are likely to be leaked!** This is due to the library authors reliance on a deterministic `__del__()` destructor. +* The memory access is read-only by design. +* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. + ## Overview @@ -33,11 +39,6 @@ For performance critical 64 bit applications, a simplified version of memory map The package was tested on all of the previously mentioned configurations. -## Limitations - -* The memory access is read-only by design. -* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. - ## Installing smmap [![Documentation Status](https://readthedocs.org/projects/smmap/badge/?version=latest)](https://readthedocs.org/projects/smmap/?badge=latest) From 30329354b370ed6bfac74290ce0c5d2ab17307d1 Mon Sep 17 00:00:00 2001 From: stuertz Date: Sun, 26 Mar 2017 15:45:10 +0200 Subject: [PATCH 0338/1392] Skip Test on Windows Currently renaming files is not supported while the the OS doesn't support renaming open files. When closing the file, as done in the code by using http://smmap.readthedocs.io/en/latest/api.html#smmap.mman.StaticWindowMapManager.force_map_handle_removal_win force_map_handle_removal_win, we can rename, but the cache does still have a handle to this file and crashes. --- gitdb/test/db/test_pack.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index a90158151..e6c2032ca 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -10,13 +10,17 @@ from gitdb.db import PackedDB from gitdb.exc import BadObject, AmbiguousObjectName +from gitdb.util import mman import os import random +import sys +from unittest import skipIf class TestPackDB(TestDBBase): + @skipIf(sys.platform == "win32", "not supported on windows currently") @with_rw_directory @with_packs_rw def test_writing(self, path): @@ -30,6 +34,11 @@ def test_writing(self, path): # packs removed - rename a file, should affect the glob pack_path = pdb.entities()[0].pack().path() new_pack_path = pack_path + "renamed" + if sys.platform == "win32": + # This is just the beginning: While using thsi function, we are not + # allowed to have any handle to thsi path, which is currently not + # the case. The pack caching does have a handle :-( + mman.force_map_handle_removal_win(pack_path) os.rename(pack_path, new_pack_path) pdb.update_cache(force=True) From 57c3a4fc59b6babe71859b2bf92b2b2fc909ce2a Mon Sep 17 00:00:00 2001 From: stuertz Date: Sun, 26 Mar 2017 15:48:07 +0200 Subject: [PATCH 0339/1392] Fixed Tests / Code for Windows. Sometimes the OS or some other process has the handle to file a bit longer, and the file could not be deleted immediatly. Retry 10 Times with 100ms distance. --- gitdb/test/performance/test_stream.py | 4 ++-- gitdb/util.py | 27 +++++++++++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 704f4d094..bd8953e15 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -9,7 +9,7 @@ from gitdb.db import LooseObjectDB from gitdb import IStream -from gitdb.util import bin_to_hex +from gitdb.util import bin_to_hex, remove from gitdb.fun import chunk_size from time import time @@ -104,5 +104,5 @@ def test_large_data_streaming(self, path): (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / (elapsed_readchunks or 1)), file=sys.stderr) # del db file so we keep something to do - os.remove(db_file) + remove(db_file) # END for each randomization factor diff --git a/gitdb/util.py b/gitdb/util.py index 242be4405..95ab9b28c 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -6,6 +6,7 @@ import os import mmap import sys +import time import errno from io import BytesIO @@ -58,7 +59,6 @@ def unpack_from(fmt, data, offset=0): isdir = os.path.isdir isfile = os.path.isfile rename = os.rename -remove = os.remove dirname = os.path.dirname basename = os.path.basename join = os.path.join @@ -67,6 +67,25 @@ def unpack_from(fmt, data, offset=0): close = os.close fsync = os.fsync + +def _retry(func, *args, **kwargs): + # Wrapper around functions, that are problematic on "Windows". Sometimes + # the OS or someone else has still a handle to the file + if sys.platform == "win32": + for _ in xrange(10): + try: + return func(*args, **kwargs) + except Exception: + time.sleep(0.1) + return func(*args, **kwargs) + else: + return func(*args, **kwargs) + + +def remove(*args, **kwargs): + return _retry(os.remove, *args, **kwargs) + + # Backwards compatibility imports from gitdb.const import ( NULL_BIN_SHA, @@ -321,7 +340,7 @@ def open(self, write=False, stream=False): self._fd = os.open(self._filepath, os.O_RDONLY | binary) except: # assure we release our lockfile - os.remove(self._lockfilepath()) + remove(self._lockfilepath()) raise # END handle lockfile # END open descriptor for reading @@ -365,7 +384,7 @@ def _end_writing(self, successful=True): # on windows, rename does not silently overwrite the existing one if sys.platform == "win32": if isfile(self._filepath): - os.remove(self._filepath) + remove(self._filepath) # END remove if exists # END win32 special handling os.rename(lockfile, self._filepath) @@ -376,7 +395,7 @@ def _end_writing(self, successful=True): chmod(self._filepath, int("644", 8)) else: # just delete the file so far, we failed - os.remove(lockfile) + remove(lockfile) # END successful handling #} END utilities From 060e5ea18b010e4d8441a5cfad699aba69593c72 Mon Sep 17 00:00:00 2001 From: stuertz Date: Sun, 26 Mar 2017 22:50:56 +0200 Subject: [PATCH 0340/1392] Release the file handle, before deleting, otherwise win fails. --- gitdb/test/performance/test_stream.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index bd8953e15..92d28e4a3 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -104,5 +104,6 @@ def test_large_data_streaming(self, path): (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / (elapsed_readchunks or 1)), file=sys.stderr) # del db file so we keep something to do + ostream = None # To release the file handle (win) remove(db_file) # END for each randomization factor From d6c097eaf759dabfd3c42b55958962b6e9a05063 Mon Sep 17 00:00:00 2001 From: stuertz Date: Mon, 27 Mar 2017 11:20:48 +0200 Subject: [PATCH 0341/1392] close smmap handles, to be able to delete files / trees in process (req. for windows) --- gitdb/pack.py | 12 ++++++++++++ gitdb/test/test_pack.py | 7 +++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index 20a451549..115d94365 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -266,6 +266,10 @@ def __init__(self, indexpath): super(PackIndexFile, self).__init__() self._indexpath = indexpath + def close(self): + mman.force_map_handle_removal_win(self._indexpath) + self._cursor = None + def _set_cache_(self, attr): if attr == "_packfile_checksum": self._packfile_checksum = self._cursor.map()[-40:-20] @@ -527,6 +531,10 @@ class PackFile(LazyMixin): def __init__(self, packpath): self._packpath = packpath + def close(self): + mman.force_map_handle_removal_win(self._packpath) + self._cursor = None + def _set_cache_(self, attr): # we fill the whole cache, whichever attribute gets queried first self._cursor = mman.make_cursor(self._packpath).use_region() @@ -668,6 +676,10 @@ def __init__(self, pack_or_index_path): self._index = self.IndexFileCls("%s.idx" % basename) # PackIndexFile instance self._pack = self.PackFileCls("%s.pack" % basename) # corresponding PackFile instance + def close(self): + self._index.close() + self._pack.close() + def _set_cache_(self, attr): # currently this can only be _offset_map # TODO: make this a simple sorted offset array which can be bisected diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 6e31363bf..24e2a3134 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -217,10 +217,11 @@ def rewind_streams(): assert os.path.getsize(ppath) > 100 # verify pack - pf = PackFile(ppath) # FIXME: Leaks file-pointer(s)! + pf = PackFile(ppath) assert pf.size() == len(pack_objs) assert pf.version() == PackFile.pack_version_default assert pf.checksum() == pack_sha + pf.close() # verify index if ipath is not None: @@ -231,6 +232,7 @@ def rewind_streams(): assert idx.packfile_checksum() == pack_sha assert idx.indexfile_checksum() == index_sha assert idx.size() == len(pack_objs) + idx.close() # END verify files exist # END for each packpath, indexpath pair @@ -245,7 +247,8 @@ def rewind_streams(): # END for each crc mode # END for each info assert count == len(pack_objs) - + entity.close() + def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets # of course without really needing such a huge pack From 891ac5126540dcb087242f9bb0cadd02ca021324 Mon Sep 17 00:00:00 2001 From: stuertz Date: Mon, 27 Mar 2017 11:36:43 +0200 Subject: [PATCH 0342/1392] Use range instead of xrange, good enough here --- gitdb/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/util.py b/gitdb/util.py index 95ab9b28c..8a1819b6d 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -72,7 +72,7 @@ def _retry(func, *args, **kwargs): # Wrapper around functions, that are problematic on "Windows". Sometimes # the OS or someone else has still a handle to the file if sys.platform == "win32": - for _ in xrange(10): + for _ in range(10): try: return func(*args, **kwargs) except Exception: From bcdffc46e3993d7743b90982009eec49df667ab4 Mon Sep 17 00:00:00 2001 From: stuertz Date: Mon, 27 Mar 2017 11:44:14 +0200 Subject: [PATCH 0343/1392] fixed to be py26 compat --- gitdb/test/db/test_pack.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index e6c2032ca..f6e275126 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -16,14 +16,16 @@ import random import sys -from unittest import skipIf +from nose.plugins.skip import SkipTest class TestPackDB(TestDBBase): - @skipIf(sys.platform == "win32", "not supported on windows currently") @with_rw_directory @with_packs_rw def test_writing(self, path): + if sys.platform == "win32": + raise SkipTest("FIXME: Currently fail on windows") + pdb = PackedDB(path) # on demand, we init our pack cache From 7bced788880015075754ce3645cef3a351166ff4 Mon Sep 17 00:00:00 2001 From: stuertz Date: Mon, 27 Mar 2017 11:46:23 +0200 Subject: [PATCH 0344/1392] Typos --- gitdb/test/db/test_pack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index f6e275126..9694238bd 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -37,9 +37,9 @@ def test_writing(self, path): pack_path = pdb.entities()[0].pack().path() new_pack_path = pack_path + "renamed" if sys.platform == "win32": - # This is just the beginning: While using thsi function, we are not - # allowed to have any handle to thsi path, which is currently not - # the case. The pack caching does have a handle :-( + # While using this function, we are not allowed to have any handle + # to this path, which is currently not the case. The pack caching + # does still have a handle :-( mman.force_map_handle_removal_win(pack_path) os.rename(pack_path, new_pack_path) From ce7889ef31c41cf5ecdb468a69f7e195b0ea5826 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 28 May 2017 17:59:03 +0200 Subject: [PATCH 0345/1392] chore(version-up): v2.0.2 Include python 3.5/3.6 in list of supported versions, provide signed archives. --- Makefile | 18 +++++++++++++++--- setup.py | 2 ++ smmap/__init__.py | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index ca3051dd4..061588245 100644 --- a/Makefile +++ b/Makefile @@ -12,13 +12,14 @@ all: $(info sdist) doc: - cd docs && make html + cd doc && make html clean-docs: - cd docs && make clean + cd doc && make clean clean-files: git clean -fx + rm -rf build/ dist/ clean: clean-files clean-docs @@ -34,4 +35,15 @@ build: sdist: ./setup.py sdist - +release: clean + # Check if latest tag is the current head we're releasing + echo "Latest tag = $$(git tag | sort -nr | head -n1)" + echo "HEAD SHA = $$(git rev-parse head)" + echo "Latest tag SHA = $$(git tag | sort -nr | head -n1 | xargs git rev-parse)" + @test "$$(git rev-parse head)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" + make force_release + +force_release: clean + git push --tags + python setup.py sdist bdist_wheel + twine upload -s -i byronimo@gmail.com dist/* diff --git a/setup.py b/setup.py index 2c519e648..800fe4fd1 100755 --- a/setup.py +++ b/setup.py @@ -50,6 +50,8 @@ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", ], long_description=long_description, tests_require=('nose', 'nosexcover'), diff --git a/smmap/__init__.py b/smmap/__init__.py index 9cfd0a1c1..dd9b0781a 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (2, 0, 1) +version_info = (2, 0, 2) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 813166d9a93eff6ea2140c2ba1714aa83feb1d4c Mon Sep 17 00:00:00 2001 From: Joseph LaFreniere Date: Wed, 7 Jun 2017 20:53:57 -0500 Subject: [PATCH 0346/1392] Add MANIFEST.in to include LICENSE in distribution From https://packaging.python.org/distributing/#manifest-in: > A "MANIFEST.in" is needed in certain cases where you need to package > additional files that python setup.py sdist (or bdist_wheel) don't > automatically include. --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..399a207e6 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +# Include the license file +include LICENSE From 91d506e120d4a0f98cbef202325e301c632445c5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 10 Jun 2017 18:49:59 +0200 Subject: [PATCH 0347/1392] chore(version-up): v2.0.3 --- smmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index dd9b0781a..b252c211c 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/Byron/smmap" -version_info = (2, 0, 2) +version_info = (2, 0, 3) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 3add9cea092514a82931108bbc05a15355900a52 Mon Sep 17 00:00:00 2001 From: wangweichen Date: Thu, 13 Jul 2017 13:55:49 +0800 Subject: [PATCH 0348/1392] fix open encoding error. --- gitdb/db/ref.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 2e3db86db..84f9f6cab 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -41,7 +41,7 @@ def _update_dbs_from_ref_file(self): # try to get as many as possible, don't fail if some are unavailable ref_paths = list() try: - with open(self._ref_file, 'r') as f: + with open(self._ref_file, 'r', encoding="utf-8") as f: ref_paths = [l.strip() for l in f] except (OSError, IOError): pass From 5e0fea5f6b9e47f52d045d0449dd1a09943496a0 Mon Sep 17 00:00:00 2001 From: wangweichen Date: Thu, 13 Jul 2017 14:34:09 +0800 Subject: [PATCH 0349/1392] change codecs to open with support encoding="utf-8" --- gitdb/db/ref.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 84f9f6cab..94a2f01f8 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -2,6 +2,7 @@ # # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php +import codecs from gitdb.db.base import ( CompoundDB, ) @@ -41,7 +42,7 @@ def _update_dbs_from_ref_file(self): # try to get as many as possible, don't fail if some are unavailable ref_paths = list() try: - with open(self._ref_file, 'r', encoding="utf-8") as f: + with codecs.open(self._ref_file, 'r', encoding="utf-8") as f: ref_paths = [l.strip() for l in f] except (OSError, IOError): pass From 0d5062eac9d03ea63975446439600e63feb83163 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 28 Sep 2017 10:52:51 +0200 Subject: [PATCH 0350/1392] Upgrade makefile --- Makefile | 16 +++++++++++++++- gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c6c159bdc..8cb323e9f 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,21 @@ SETUP = $(PYTHON) setup.py TESTRUNNER = $(shell which nosetests) TESTFLAGS = -all: build +all:: + @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all + +release:: clean + # Check if latest tag is the current head we're releasing + echo "Latest tag = $$(git tag | sort -nr | head -n1)" + echo "HEAD SHA = $$(git rev-parse head)" + echo "Latest tag SHA = $$(git tag | sort -nr | head -n1 | xargs git rev-parse)" + @test "$$(git rev-parse head)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" + make force_release + +force_release:: clean + git push --tags + python3 setup.py sdist bdist_wheel + twine upload -s -i byronimo@gmail.com dist/* doc:: make -C doc/ html diff --git a/gitdb/__init__.py b/gitdb/__init__.py index bfa083bb3..e184e4b1f 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 0) +version_info = (2, 0, 3) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index ac5df7061..91d506e12 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit ac5df7061ee11232346b3d0eb3aa5b43eebc847d +Subproject commit 91d506e120d4a0f98cbef202325e301c632445c5 diff --git a/setup.py b/setup.py index f7e3761ab..4bff11dff 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 0) +version_info = (2, 0, 3) __version__ = '.'.join(str(i) for i in version_info) setup( From ed3ecbbe7b05433f6c70410a3f1fda1ae85508a3 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 3 Aug 2017 09:46:50 -0500 Subject: [PATCH 0351/1392] Update homepage --- smmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index b252c211c..725c420ea 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -2,7 +2,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" -__homepage__ = "https://github.com/Byron/smmap" +__homepage__ = "https://github.com/gitpython-developers/smmap" version_info = (2, 0, 3) __version__ = '.'.join(str(i) for i in version_info) From 90c4f25493b918ff9dc4ee52ae8216a554bb3446 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 29 Sep 2017 13:16:49 +0200 Subject: [PATCH 0352/1392] Add python 3.6 to the testing suite --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ba0beaa7b..7288a2acb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "3.3" - "3.4" - "3.5" + - "3.6" # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) git: From a4ca69e9c93d00185dc729e68f2ef86ffe138410 Mon Sep 17 00:00:00 2001 From: Michael Overmeyer Date: Mon, 5 Mar 2018 20:06:45 +0000 Subject: [PATCH 0353/1392] Switched broken pypip.in badges to shields.io --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ca51dfaba..917b403b0 100644 --- a/README.rst +++ b/README.rst @@ -6,10 +6,10 @@ GitDB allows you to access bare git repositories for reading and writing. It aim Installation ============ -.. image:: https://pypip.in/version/gitdb/badge.svg +.. image:: https://img.shields.io/pypi/v/gitdb.svg :target: https://pypi.python.org/pypi/gitdb/ :alt: Latest Version -.. image:: https://pypip.in/py_versions/gitdb/badge.svg +.. image:: https://img.shields.io/pypi/pyversions/gitdb.svg :target: https://pypi.python.org/pypi/gitdb/ :alt: Supported Python versions .. image:: https://readthedocs.org/projects/gitdb/badge/?version=latest From 857196a7312209eed5c84fca857ab6c486bcbd6b Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 7 Sep 2018 21:29:54 +0300 Subject: [PATCH 0354/1392] Drop support for EOL Python --- .travis.yml | 2 -- gitdb/util.py | 5 +---- setup.py | 4 +--- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7288a2acb..1341a1d9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,6 @@ language: python python: - - "2.6" - "2.7" - - "3.3" - "3.4" - "3.5" - "3.6" diff --git a/gitdb/util.py b/gitdb/util.py index 8a1819b6d..d680f9766 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -19,10 +19,7 @@ # initialize our global memory manager instance # Use it to free cached (and unused) resources. -if sys.version_info < (2, 6): - mman = StaticWindowMapManager() -else: - mman = SlidingWindowMapManager() +mman = SlidingWindowMapManager() # END handle mman import hashlib diff --git a/setup.py b/setup.py index 4bff11dff..931596472 100755 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ zip_safe=False, install_requires=['smmap2 >= 2.0.0'], long_description="""GitDB is a pure-Python git object database""", + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 5 - Production/Stable", @@ -34,11 +35,8 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", - "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", ] From 140104a7ecb3dfb5c43b57c4829b9d7142b4aeaf Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 7 Sep 2018 21:32:56 +0300 Subject: [PATCH 0355/1392] Upgrade Python syntax with pyupgrade https://github.com/asottile/pyupgrade --- gitdb/db/pack.py | 2 +- gitdb/db/ref.py | 2 +- gitdb/test/lib.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 6b03d8383..1e37d738e 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -148,7 +148,7 @@ def update_cache(self, force=False): # packs are supposed to be prefixed with pack- by git-convention # get all pack files, figure out what changed pack_files = set(glob.glob(os.path.join(self.root_path(), "pack-*.pack"))) - our_pack_files = set(item[1].pack().path() for item in self._entities) + our_pack_files = {item[1].pack().path() for item in self._entities} # new packs for pack_file in (pack_files - our_pack_files): diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 94a2f01f8..2bb1de790 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -49,7 +49,7 @@ def _update_dbs_from_ref_file(self): # END handle alternates ref_paths_set = set(ref_paths) - cur_ref_paths_set = set(db.root_path() for db in self._dbs) + cur_ref_paths_set = {db.root_path() for db in self._dbs} # remove existing for path in (cur_ref_paths_set - ref_paths_set): diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index bbdd241cd..ab1842dbd 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -86,7 +86,7 @@ def wrapper(self): try: return func(self, path) except Exception: - sys.stderr.write("Test %s.%s failed, output is at %r\n" % (type(self).__name__, func.__name__, path)) + sys.stderr.write("Test {}.{} failed, output is at {!r}\n".format(type(self).__name__, func.__name__, path)) keep = True raise finally: From 186db667f703b695c2914040e98d7977f6e61272 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 7 Sep 2018 21:36:22 +0300 Subject: [PATCH 0356/1392] Python 3.6 is tested and supported --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 931596472..27bb75429 100755 --- a/setup.py +++ b/setup.py @@ -39,5 +39,6 @@ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", ] ) From 3e28e6e896543814ea312baa9ba6c6065caba797 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 13 Oct 2018 12:41:52 +0200 Subject: [PATCH 0357/1392] Bump patch level: remove support for old python versions --- gitdb/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index e184e4b1f..344c3de01 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 3) +version_info = (2, 0, 4) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 27bb75429..ea6ba189f 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 3) +version_info = (2, 0, 4) __version__ = '.'.join(str(i) for i in version_info) setup( From b1adf606f416f82ec69cd83cfc2b94cddc6928bd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 13 Oct 2018 12:45:29 +0200 Subject: [PATCH 0358/1392] Another version bump... dunno what happened there. --- gitdb/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 344c3de01..a2d26245b 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 4) +version_info = (2, 0, 5) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index ea6ba189f..27eb65c8e 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 4) +version_info = (2, 0, 5) __version__ = '.'.join(str(i) for i in version_info) setup( From 54a34e8976587762df7b02ef09b0150c065fb9e1 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 7 Sep 2018 22:04:27 +0300 Subject: [PATCH 0359/1392] Drop support for EOL Python --- .travis.yml | 5 ----- README.md | 3 +-- doc/source/intro.rst | 3 +-- setup.py | 6 ++---- smmap/buf.py | 43 ++++++++++++++----------------------------- smmap/util.py | 38 +------------------------------------- tox.ini | 2 +- 7 files changed, 20 insertions(+), 80 deletions(-) diff --git a/.travis.yml b/.travis.yml index 464fafb2e..ff77ede15 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,6 @@ language: python python: - # These versions are unsupported by travis, even though smmap claims to still support these outdated versions - # - 2.4 - # - 2.5 - - 2.6 - 2.7 - - 3.3 - 3.4 - 3.5 install: diff --git a/README.md b/README.md index 247848d04..08702930e 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ Although memory maps have many advantages, they represent a very limited system * **System resources (file-handles) are likely to be leaked!** This is due to the library authors reliance on a deterministic `__del__()` destructor. * The memory access is read-only by design. -* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. ## Overview @@ -34,7 +33,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 2.4, 2.5, 2.6, 2.7 or 3.3 +* Python 2.7 or 3.4+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 15f5bf08a..8f7cf3c26 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -22,7 +22,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 2.4, 2.5, 2.6, 2.7 or 3.3 +* Python 2.7 or 3.4+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. @@ -31,7 +31,6 @@ The package was tested on all of the previously mentioned configurations. Limitations ########### * The memory access is read-only by design. -* In python below 2.6, memory maps will be created in compatibility mode which works, but creates inefficient memory mappings as they always start at offset 0. ################ Installing smmap diff --git a/setup.py b/setup.py index 800fe4fd1..23347a3e3 100755 --- a/setup.py +++ b/setup.py @@ -13,12 +13,12 @@ if os.path.exists("README.md"): long_description = codecs.open('README.md', "r", "utf-8").read() else: - long_description = "See http://github.com/gitpython-developers/smmap" + long_description = "See https://github.com/gitpython-developers/smmap" setup( name="smmap2", version=smmap.__version__, - description="A pure python implementation of a sliding window memory map manager", + description="A pure Python implementation of a sliding window memory map manager", author=smmap.__author__, author_email=smmap.__contact__, url=smmap.__homepage__, @@ -45,10 +45,8 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", diff --git a/smmap/buf.py b/smmap/buf.py index 438292b60..786775a8b 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -88,35 +88,20 @@ def __getslice__(self, i, j): # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower # in the previous iteration of this code pyvers = sys.version_info[:2] - if (3, 0) <= pyvers <= (3, 3): - # Memory view cannot be joined below python 3.4 ... - out = bytes() - while l: - c.use_region(ofs, l) - assert c.is_valid() - d = c.buffer()[:l] - ofs += len(d) - l -= len(d) - # This is slower than the join ... but what can we do ... - out += d - del(d) - # END while there are bytes to read - return out - else: - md = list() - while l: - c.use_region(ofs, l) - assert c.is_valid() - d = c.buffer()[:l] - ofs += len(d) - l -= len(d) - # Make sure we don't keep references, as c.use_region() might attempt to free resources, but - # can't unless we use pure bytes - if hasattr(d, 'tobytes'): - d = d.tobytes() - md.append(d) - # END while there are bytes to read - return bytes().join(md) + md = list() + while l: + c.use_region(ofs, l) + assert c.is_valid() + d = c.buffer()[:l] + ofs += len(d) + l -= len(d) + # Make sure we don't keep references, as c.use_region() might attempt to free resources, but + # can't unless we use pure bytes + if hasattr(d, 'tobytes'): + d = d.tobytes() + md.append(d) + # END while there are bytes to read + return bytes().join(md) # END fast or slow path #{ Interface diff --git a/smmap/util.py b/smmap/util.py index 02df41a13..defef1f32 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -3,13 +3,7 @@ import sys from mmap import mmap, ACCESS_READ -try: - from mmap import ALLOCATIONGRANULARITY -except ImportError: - # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly - # useful for aligning the offset. The offset argument doesn't exist there though - from mmap import PAGESIZE as ALLOCATIONGRANULARITY -# END handle pythons missing quality assurance +from mmap import ALLOCATIONGRANULARITY __all__ = ["align_to_mmap", "is_64_bit", "buffer", "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] @@ -116,11 +110,6 @@ class MapRegion(object): '_size', # cached size of our memory map '__weakref__' ] - _need_compat_layer = sys.version_info[:2] < (2, 6) - - if _need_compat_layer: - __slots__.append('_mfb') # mapped memory buffer to provide offset - # END handle additional slot #{ Configuration #} END configuration @@ -147,11 +136,6 @@ def __init__(self, path_or_fd, ofs, size, flags=0): kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size sizeofs = ofs - if self._need_compat_layer: - del(kwargs['offset']) - corrected_size += ofs - sizeofs = 0 - # END handle python not supporting offset ! Arg # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because @@ -161,10 +145,6 @@ def __init__(self, path_or_fd, ofs, size, flags=0): # END handle memory mode self._size = len(self._mf) - - if self._need_compat_layer: - self._mfb = buffer(self._mf, ofs, self._size) - # END handle buffer wrapping finally: if isinstance(path_or_fd, string_types()): os.close(fd) @@ -224,22 +204,6 @@ def release(self): """Release all resources this instance might hold. Must only be called if there usage_count() is zero""" self._mf.close() - # re-define all methods which need offset adjustments in compatibility mode - if _need_compat_layer: - def size(self): - return self._size - self._b - - def ofs_end(self): - # always the size - we are as large as it gets - return self._size - - def buffer(self): - return self._mfb - - def includes_ofs(self, ofs): - return self._b <= ofs < self._size - # END handle compat layer - #} END interface diff --git a/tox.ini b/tox.ini index 35b813753..74ca3481d 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py26, py27, py33, py34 +envlist = flake8, py27, py34 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From 68a94764aeffe9b90f82f82bbf7d166cfdfacc87 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 7 Sep 2018 22:07:12 +0300 Subject: [PATCH 0360/1392] Add python_requires to help pip --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 23347a3e3..9136339ae 100755 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", classifiers=[ # Picked from # http://pypi.python.org/pypi?:action=list_classifiers From 48e9e30b0ef3c24ac7ed88e6e3bfa37dc945bf4c Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 7 Sep 2018 22:07:54 +0300 Subject: [PATCH 0361/1392] Test Python 3.6 --- .travis.yml | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ff77ede15..9cccb75cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - 2.7 - 3.4 - 3.5 + - 3.6 install: - pip install coveralls script: diff --git a/tox.ini b/tox.ini index 74ca3481d..d1f558bc0 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py27, py34 +envlist = flake8, py27, py34, py35, py36 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From 53944ab34f18d05135ab4b507e718eccfe699248 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 13 Oct 2018 12:56:00 +0200 Subject: [PATCH 0362/1392] bump patch; remove old python versions --- smmap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index 725c420ea..1728b832f 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (2, 0, 3) +version_info = (2, 0, 5) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 2f075234b84511ee63b74e69ccc8ddc331de1199 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Mon, 15 Oct 2018 10:51:19 +0100 Subject: [PATCH 0363/1392] Tell PyPI long_description is Markdown Rendering of the README on PyPI is broken because it expects RST by default and Markdown is being uploaded. This fixes it. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 9136339ae..09755dac5 100755 --- a/setup.py +++ b/setup.py @@ -53,6 +53,7 @@ "Programming Language :: Python :: 3.6", ], long_description=long_description, + long_description_content_type='text/markdown', tests_require=('nose', 'nosexcover'), test_suite='nose.collector' ) From 0e4b57d4511686d1aeb5479a7aa0dad3a5338d6e Mon Sep 17 00:00:00 2001 From: xarx00 Date: Fri, 5 Apr 2019 10:46:53 +0200 Subject: [PATCH 0364/1392] Fix for UnicodeEncodeError in git.Repo.clone_from() when path contains non-ascii characters --- gitdb/utils/encoding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 4d270af9d..d8fd59a4a 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -8,7 +8,7 @@ text_type = unicode -def force_bytes(data, encoding="ascii"): +def force_bytes(data, encoding="utf-8"): if isinstance(data, bytes): return data From a0060cfdc9166bb0b3104e8015faf0689aa6daf1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 20 Jul 2019 18:45:04 +0800 Subject: [PATCH 0365/1392] Associate smmap2 on pypi with this repository Fixes #43 --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 08702930e..c7a38173c 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,10 @@ Issues can be filed on github: * https://github.com/gitpython-developers/smmap/issues +A link to the pypi page related to this repository: + +* https://pypi.org/project/smmap2/ + ## License Information From 79b705f061b51dc151a00729b722fbdebde59f5c Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 25 Sep 2019 21:00:30 +0300 Subject: [PATCH 0366/1392] loose: rename only if needed Our user was experiencing issue [1] when using a git repository on NTFS mount running on Linux. The current check checks if we are running on Windows, but it should really check if we are on NTFS. And since checking fs type is not that trivial and not efficient, it is simpler and better to just always apply NTFS-specific logic, since it works on other filesystems as well. [1] https://github.com/iterative/dvc/issues/1880#issuecomment-483253764 --- gitdb/db/loose.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 192c524af..53ade9f79 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -225,16 +225,12 @@ def store(self, istream): if not isdir(obj_dir): mkdir(obj_dir) # END handle destination directory - # rename onto existing doesn't work on windows - if os.name == 'nt': - if isfile(obj_path): - remove(tmp_path) - else: - rename(tmp_path, obj_path) - # end rename only if needed + # rename onto existing doesn't work on NTFS + if isfile(obj_path): + remove(tmp_path) else: rename(tmp_path, obj_path) - # END handle win32 + # end rename only if needed # make sure its readable for all ! It started out as rw-- tmp file # but needs to be rwrr From d77bd023a61419effe77184c52ccf3e19afa6f60 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 28 Sep 2019 13:21:35 +0200 Subject: [PATCH 0367/1392] Bump version --- gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index e184e4b1f..344c3de01 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 3) +version_info = (2, 0, 4) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 91d506e12..a0060cfdc 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 91d506e120d4a0f98cbef202325e301c632445c5 +Subproject commit a0060cfdc9166bb0b3104e8015faf0689aa6daf1 diff --git a/setup.py b/setup.py index 27bb75429..ea6ba189f 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 3) +version_info = (2, 0, 4) __version__ = '.'.join(str(i) for i in version_info) setup( From 43e16318e9ab95f146e230afe0a7cbdc848473fe Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 28 Sep 2019 13:28:50 +0200 Subject: [PATCH 0368/1392] bump version again... --- gitdb/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 344c3de01..4e884076b 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 4) +version_info = (2, 0, 6) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index ea6ba189f..6a2174fce 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 4) +version_info = (2, 0, 6) __version__ = '.'.join(str(i) for i in version_info) setup( From 7729239951b5561f5bb5c2d5152ff76833b11826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Wed, 8 Jan 2020 09:36:55 +0100 Subject: [PATCH 0369/1392] Fix deprecated calls for Python 3.9 The array methods fromstring/tostring have been deprecated since Python 3.2. Python 3.9 removes them completely. This was discovered when trying to build gitdb package for Fedora 33. https://bugzilla.redhat.com/show_bug.cgi?id=1788660 --- gitdb/pack.py | 2 +- gitdb/test/lib.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index 115d94365..2ad2324eb 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -410,7 +410,7 @@ def offsets(self): if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears - a.fromstring(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) + a.frombytes(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) # networkbyteorder to something array likes more if sys.byteorder == 'little': diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index ab1842dbd..42b9ddc7e 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -157,7 +157,7 @@ def make_bytes(size_in_bytes, randomize=False): random.shuffle(producer) # END randomize a = array('i', producer) - return a.tostring() + return a.tobytes() def make_object(type, data): From c880f6b0550770eee559091d6276a2e2b097a83a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 15 Feb 2020 09:41:05 +0800 Subject: [PATCH 0370/1392] don't test python 2.7 anymore, support is dropped --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1341a1d9d..17d63804d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "2.7" - "3.4" - "3.5" - "3.6" From 2f9a799a6c9d125012bb09473bbfe6110f2a7391 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 15 Feb 2020 09:59:38 +0800 Subject: [PATCH 0371/1392] remove appveyor It is slow, it fails, and windows support seems unmaintained, besides always having been an incredible time sink. Thanks to everyone who brought GitDb to where it is right now, and I am happy to bring windows testing back if a maintainer can be found. --- .appveyor.yml | 49 ------------------------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 2daadaa4d..000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,49 +0,0 @@ -# CI on Windows via appveyor -environment: - - matrix: - ## MINGW - # - - PYTHON: "C:\\Python27" - PYTHON_VERSION: "2.7" - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4" - - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5" - - PYTHON: "C:\\Miniconda35-x64" - PYTHON_VERSION: "3.5" - IS_CONDA: "yes" - -install: - - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% - - ## Print configuration for debugging. - # - - | - echo %PATH% - uname -a - where python pip pip2 pip3 pip34 - python --version - python -c "import struct; print(struct.calcsize('P') * 8)" - - - IF "%IS_CONDA%"=="yes" ( - conda info -a & - conda install --yes --quiet pip - ) - - pip install nose wheel coveralls - - ## For commits performed with the default user. - - | - git config --global user.email "travis@ci.com" - git config --global user.name "Travis Runner" - - - pip install -e . - -build: false - -test_script: - - IF "%PYTHON_VERSION%"=="3.5" ( - nosetests -v --with-coverage - ) ELSE ( - nosetests -v - ) From df73d7f6874ff11be1b09f65c8dc425671bb924e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 15 Feb 2020 10:01:11 +0800 Subject: [PATCH 0372/1392] Release 3.0.0 --- gitdb/__init__.py | 2 +- setup.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 4e884076b..c9c827970 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 6) +version_info = (3, 0, 0) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 6a2174fce..a66267e48 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (2, 0, 6) +version_info = (3, 0, 0) __version__ = '.'.join(str(i) for i in version_info) setup( @@ -22,7 +22,7 @@ zip_safe=False, install_requires=['smmap2 >= 2.0.0'], long_description="""GitDB is a pure-Python git object database""", - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + python_requires='>=3.4', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 5 - Production/Stable", @@ -34,11 +34,10 @@ "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7" ] ) From e6ee8bf864c726a5461600de28d64c1f06f4e163 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 15 Feb 2020 10:03:50 +0800 Subject: [PATCH 0373/1392] Change package signature to the only yubikey I can use right now --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8cb323e9f..82c0a3bc8 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ release:: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i byronimo@gmail.com dist/* + twine upload -s -i 763629FEC8788FC35128B5F6EE029D1E5EB40300 dist/* doc:: make -C doc/ html From d6d1550a1e8dc327d5b310228a66f25a59d6ce9f Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 15 Feb 2020 14:19:38 -0600 Subject: [PATCH 0374/1392] Remove badges for no longer existing Issue Stats site from README --- README.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.rst b/README.rst index 917b403b0..9febff0ed 100644 --- a/README.rst +++ b/README.rst @@ -56,10 +56,6 @@ DEVELOPMENT :target: https://ci.appveyor.com/project/ankostis/gitpython/branch/master) .. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png :target: https://coveralls.io/r/gitpython-developers/gitdb -.. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/pr - :target: http://www.issuestats.com/github/gitpython-developers/gitdb -.. image:: http://www.issuestats.com/github/gitpython-developers/gitdb/badge/issue - :target: http://www.issuestats.com/github/gitpython-developers/gitdb The library is considered mature, and not under active development. It's primary (known) use is in git-python. From 58bce6bd1051e4fd5df7c5c8123a1783cc6e9f84 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:06:35 -0600 Subject: [PATCH 0375/1392] Remove and replace compat.izip --- gitdb/fun.py | 4 ++-- gitdb/pack.py | 3 +-- gitdb/utils/compat.py | 2 -- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index 8ca38c867..3a2248fa1 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -16,7 +16,7 @@ from gitdb.const import NULL_BYTE, BYTE_SPACE from gitdb.utils.encoding import force_text -from gitdb.utils.compat import izip, buffer, xrange, PY3 +from gitdb.utils.compat import buffer, xrange, PY3 from gitdb.typ import ( str_blob_type, str_commit_type, @@ -314,7 +314,7 @@ def check_integrity(self, target_size=-1): right.next() # this is very pythonic - we might have just use index based access here, # but this could actually be faster - for lft, rgt in izip(left, right): + for lft, rgt in zip(left, right): assert lft.rbound() == rgt.to assert lft.to + lft.ts == rgt.to # END for each pair diff --git a/gitdb/pack.py b/gitdb/pack.py index 2ad2324eb..748df3880 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -63,7 +63,6 @@ from gitdb.const import NULL_BYTE from gitdb.utils.compat import ( - izip, buffer, xrange, to_bytes @@ -696,7 +695,7 @@ def _set_cache_(self, attr): iter_offsets = iter(offsets_sorted) iter_offsets_plus_one = iter(offsets_sorted) next(iter_offsets_plus_one) - consecutive = izip(iter_offsets, iter_offsets_plus_one) + consecutive = zip(iter_offsets, iter_offsets_plus_one) offset_map = dict(consecutive) diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index a7899cb14..586f3bb9d 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -3,11 +3,9 @@ PY3 = sys.version_info[0] == 3 try: - from itertools import izip xrange = xrange except ImportError: # py3 - izip = zip xrange = range # end handle python version From 73a9f7965139e319446c04bbcc9794a8db0de45a Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:07:51 -0600 Subject: [PATCH 0376/1392] Remove and replace izip in TestPack --- gitdb/test/test_pack.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 24e2a3134..dd1c8302b 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -27,11 +27,6 @@ from gitdb.util import to_bin_sha from gitdb.utils.compat import xrange -try: - from itertools import izip -except ImportError: - izip = zip - from nose import SkipTest import os @@ -155,7 +150,7 @@ def test_pack_entity(self, rw_dir): pack_objs.extend(entity.stream_iter()) count = 0 - for info, stream in izip(entity.info_iter(), entity.stream_iter()): + for info, stream in zip(entity.info_iter(), entity.stream_iter()): count += 1 assert info.binsha == stream.binsha assert len(info.binsha) == 20 From 4a692fdd43e67810509b1c1843fa203714b14f0e Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:13:00 -0600 Subject: [PATCH 0377/1392] Remove and replace compat.xrange --- gitdb/db/pack.py | 3 +-- gitdb/fun.py | 4 ++-- gitdb/pack.py | 9 ++++----- gitdb/test/db/lib.py | 3 +-- gitdb/test/lib.py | 3 +-- gitdb/test/performance/test_pack.py | 3 +-- gitdb/test/test_pack.py | 3 +-- gitdb/utils/compat.py | 7 ------- 8 files changed, 11 insertions(+), 24 deletions(-) diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 1e37d738e..177ed7bb2 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -18,7 +18,6 @@ ) from gitdb.pack import PackEntity -from gitdb.utils.compat import xrange from functools import reduce @@ -107,7 +106,7 @@ def sha_iter(self): for entity in self.entities(): index = entity.index() sha_by_index = index.sha - for index in xrange(index.size()): + for index in range(index.size()): yield sha_by_index(index) # END for each index # END for each entity diff --git a/gitdb/fun.py b/gitdb/fun.py index 3a2248fa1..7203de978 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -16,7 +16,7 @@ from gitdb.const import NULL_BYTE, BYTE_SPACE from gitdb.utils.encoding import force_text -from gitdb.utils.compat import buffer, xrange, PY3 +from gitdb.utils.compat import buffer, PY3 from gitdb.typ import ( str_blob_type, str_commit_type, @@ -264,7 +264,7 @@ def compress(self): # if first_data_index is not None: nd = StringIO() # new data so = self[first_data_index].to # start offset in target buffer - for x in xrange(first_data_index, i - 1): + for x in range(first_data_index, i - 1): xdc = self[x] nd.write(xdc.data[:xdc.ts]) # END collect data diff --git a/gitdb/pack.py b/gitdb/pack.py index 748df3880..f01055493 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -64,7 +64,6 @@ from gitdb.const import NULL_BYTE from gitdb.utils.compat import ( buffer, - xrange, to_bytes ) @@ -206,7 +205,7 @@ def write(self, pack_sha, write): for t in self._objs: tmplist[byte_ord(t[0][0])] += 1 # END prepare fanout - for i in xrange(255): + for i in range(255): v = tmplist[i] sha_write(pack('>L', v)) tmplist[i + 1] += v @@ -375,7 +374,7 @@ def _read_fanout(self, byte_offset): d = self._cursor.map() out = list() append = out.append - for i in xrange(256): + for i in range(256): append(unpack_from('>L', d, byte_offset + i * 4)[0]) # END for each entry return out @@ -416,7 +415,7 @@ def offsets(self): a.byteswap() return a else: - return tuple(self.offset(index) for index in xrange(self.size())) + return tuple(self.offset(index) for index in range(self.size())) # END handle version def sha_to_index(self, sha): @@ -715,7 +714,7 @@ def _iter_objects(self, as_stream): """Iterate over all objects in our index and yield their OInfo or OStream instences""" _sha = self._index.sha _object = self._object - for index in xrange(self._index.size()): + for index in range(self._index.size()): yield _object(_sha(index), as_stream, index) # END for each index diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 528bcc144..c6f4316cd 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -23,7 +23,6 @@ from gitdb.exc import BadObject from gitdb.typ import str_blob_type -from gitdb.utils.compat import xrange from io import BytesIO @@ -45,7 +44,7 @@ def _assert_object_writing_simple(self, db): # write a bunch of objects and query their streams and info null_objs = db.size() ni = 250 - for i in xrange(ni): + for i in range(ni): data = pack(">L", i) istream = IStream(str_blob_type, len(data), BytesIO(data)) new_istream = db.store(istream) diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 42b9ddc7e..a04084f5f 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -4,7 +4,6 @@ # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Utilities used in ODB testing""" from gitdb import OStream -from gitdb.utils.compat import xrange import sys import random @@ -151,7 +150,7 @@ def make_bytes(size_in_bytes, randomize=False): """:return: string with given size in bytes :param randomize: try to produce a very random stream""" actual_size = size_in_bytes // 4 - producer = xrange(actual_size) + producer = range(actual_size) if randomize: producer = list(producer) random.shuffle(producer) diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index fc8d9d54f..b59d5a971 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -17,7 +17,6 @@ from gitdb.typ import str_blob_type from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB -from gitdb.utils.compat import xrange from gitdb.test.lib import skip_on_travis_ci import sys @@ -118,7 +117,7 @@ def test_correctness(self): for entity in pdb.entities(): pack_verify = entity.is_valid_stream sha_by_index = entity.index().sha - for index in xrange(entity.index().size()): + for index in range(entity.index().size()): try: assert pack_verify(sha_by_index(index), use_crc=crc) count += 1 diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index dd1c8302b..8bf78f0d2 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -25,7 +25,6 @@ from gitdb.fun import delta_types from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha -from gitdb.utils.compat import xrange from nose import SkipTest @@ -58,7 +57,7 @@ def _assert_index_file(self, index, version, size): assert len(index.offsets()) == size # get all data of all objects - for oidx in xrange(index.size()): + for oidx in range(index.size()): sha = index.sha(oidx) assert oidx == index.sha_to_index(sha) diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index 586f3bb9d..99e7ae654 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -2,13 +2,6 @@ PY3 = sys.version_info[0] == 3 -try: - xrange = xrange -except ImportError: - # py3 - xrange = range -# end handle python version - try: # Python 2 buffer = buffer From 3d14cc84b0a5e42d755d24d77a6e4af6bea8c3c1 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:36:03 -0600 Subject: [PATCH 0378/1392] Remove and replace compat.buffer --- gitdb/fun.py | 8 ++++---- gitdb/pack.py | 11 ++++------- gitdb/stream.py | 5 ++--- gitdb/utils/compat.py | 10 ---------- 4 files changed, 10 insertions(+), 24 deletions(-) diff --git a/gitdb/fun.py b/gitdb/fun.py index 7203de978..92b8b1c70 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -16,7 +16,7 @@ from gitdb.const import NULL_BYTE, BYTE_SPACE from gitdb.utils.encoding import force_text -from gitdb.utils.compat import buffer, PY3 +from gitdb.utils.compat import PY3 from gitdb.typ import ( str_blob_type, str_commit_type, @@ -101,7 +101,7 @@ def delta_chunk_apply(dc, bbuf, write): :param write: write method to call with data to write""" if dc.data is None: # COPY DATA FROM SOURCE - write(buffer(bbuf, dc.so, dc.ts)) + write(bbuf[dc.so:dc.so + dc.ts]) else: # APPEND DATA # whats faster: if + 4 function calls or just a write with a slice ? @@ -698,7 +698,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): if (rbound < cp_size or rbound > src_buf_size): break - write(buffer(src_buf, cp_off, cp_size)) + write(src_buf[cp_off:cp_off + cp_size]) elif c: write(db[i:i + c]) i += c @@ -741,7 +741,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): if (rbound < cp_size or rbound > src_buf_size): break - write(buffer(src_buf, cp_off, cp_size)) + write(src_buf[cp_off:cp_off + cp_size]) elif c: write(db[i:i + c]) i += c diff --git a/gitdb/pack.py b/gitdb/pack.py index f01055493..68da2b7fb 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -62,10 +62,7 @@ from binascii import crc32 from gitdb.const import NULL_BYTE -from gitdb.utils.compat import ( - buffer, - to_bytes -) +from gitdb.utils.compat import to_bytes import tempfile import array @@ -117,7 +114,7 @@ def pack_object_at(cursor, offset, as_stream): # END handle type id abs_data_offset = offset + total_rela_offset if as_stream: - stream = DecompressMemMapReader(buffer(data, total_rela_offset), False, uncomp_size) + stream = DecompressMemMapReader(data[total_rela_offset:], False, uncomp_size) if delta_info is None: return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream) else: @@ -408,7 +405,7 @@ def offsets(self): if self._version == 2: # read stream to array, convert to tuple a = array.array('I') # 4 byte unsigned int, long are 8 byte on 64 bit it appears - a.frombytes(buffer(self._cursor.map(), self._pack_offset, self._pack_64_offset - self._pack_offset)) + a.frombytes(self._cursor.map()[self._pack_offset:self._pack_64_offset]) # networkbyteorder to something array likes more if sys.byteorder == 'little': @@ -836,7 +833,7 @@ def is_valid_stream(self, sha, use_crc=False): while cur_pos < next_offset: rbound = min(cur_pos + chunk_size, next_offset) size = rbound - cur_pos - this_crc_value = crc_update(buffer(pack_data, cur_pos, size), this_crc_value) + this_crc_value = crc_update(pack_data[cur_pos:cur_pos + size], this_crc_value) cur_pos += size # END window size loop diff --git a/gitdb/stream.py b/gitdb/stream.py index 2f4c12dfb..b94ef245d 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -27,7 +27,6 @@ ) from gitdb.const import NULL_BYTE, BYTE_SPACE -from gitdb.utils.compat import buffer from gitdb.utils.encoding import force_bytes has_perf_mod = False @@ -278,7 +277,7 @@ def read(self, size=-1): # END adjust winsize # takes a slice, but doesn't copy the data, it says ... - indata = buffer(self._m, self._cws, self._cwe - self._cws) + indata = self._m[self._cws:self._cwe] # get the actual window end to be sure we don't use it for computations self._cwe = self._cws + len(indata) @@ -414,7 +413,7 @@ def _set_cache_brute_(self, attr): buf = dstream.read(512) # read the header information + X offset, src_size = msb_size(buf) offset, target_size = msb_size(buf, offset) - buffer_info_list.append((buffer(buf, offset), offset, src_size, target_size)) + buffer_info_list.append((buf[offset:], offset, src_size, target_size)) max_target_size = max(max_target_size, target_size) # END for each delta stream diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index 99e7ae654..8c7c06bd7 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -4,22 +4,12 @@ try: # Python 2 - buffer = buffer memoryview = buffer # Assume no memory view ... def to_bytes(i): return i except NameError: # Python 3 has no `buffer`; only `memoryview` - # However, it's faster to just slice the object directly, maybe it keeps a view internally - def buffer(obj, offset, size=None): - if size is None: - # return memoryview(obj)[offset:] - return obj[offset:] - else: - # return memoryview(obj)[offset:offset+size] - return obj[offset:offset + size] - # end buffer reimplementation # smmap can return memory view objects, which can't be compared as buffers/bytes can ... def to_bytes(i): if isinstance(i, memoryview): From d1c20d559a1f264dad12d1033a052ffc1c159260 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:37:34 -0600 Subject: [PATCH 0379/1392] Remove compat.memoryview --- gitdb/utils/compat.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index 8c7c06bd7..d5791edcf 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -4,20 +4,15 @@ try: # Python 2 - memoryview = buffer - # Assume no memory view ... def to_bytes(i): return i except NameError: - # Python 3 has no `buffer`; only `memoryview` # smmap can return memory view objects, which can't be compared as buffers/bytes can ... def to_bytes(i): if isinstance(i, memoryview): return i.tobytes() return i - memoryview = memoryview - try: MAXSIZE = sys.maxint except AttributeError: From 59c053ee05d6e5f50f8699260aa0e362b567c033 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:54:24 -0600 Subject: [PATCH 0380/1392] Remove and replace compat.to_bytes --- gitdb/pack.py | 7 +++++-- gitdb/utils/compat.py | 11 ----------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index 68da2b7fb..a38468e37 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -62,7 +62,6 @@ from binascii import crc32 from gitdb.const import NULL_BYTE -from gitdb.utils.compat import to_bytes import tempfile import array @@ -877,7 +876,11 @@ def collect_streams_at_offset(self, offset): stream = streams[-1] while stream.type_id in delta_types: if stream.type_id == REF_DELTA: - sindex = self._index.sha_to_index(to_bytes(stream.delta_info)) + # smmap can return memory view objects, which can't be compared as buffers/bytes can ... + if isinstance(stream.delta_info, memoryview): + sindex = self._index.sha_to_index(stream.delta_info.tobytes()) + else: + sindex = self._index.sha_to_index(stream.delta_info) if sindex is None: break stream = self._pack.stream(self._index.offset(sindex)) diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index d5791edcf..6909c53e0 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -2,17 +2,6 @@ PY3 = sys.version_info[0] == 3 -try: - # Python 2 - def to_bytes(i): - return i -except NameError: - # smmap can return memory view objects, which can't be compared as buffers/bytes can ... - def to_bytes(i): - if isinstance(i, memoryview): - return i.tobytes() - return i - try: MAXSIZE = sys.maxint except AttributeError: From 321c3b46b4792cf83bf0c5814d5a1a43cdf6933d Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 07:57:23 -0600 Subject: [PATCH 0381/1392] Remove and replace compat.MAXSIZE --- gitdb/db/loose.py | 4 ++-- gitdb/utils/compat.py | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 53ade9f79..7bf92dacf 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -50,11 +50,11 @@ stream_copy ) -from gitdb.utils.compat import MAXSIZE from gitdb.utils.encoding import force_bytes import tempfile import os +import sys __all__ = ('LooseObjectDB', ) @@ -196,7 +196,7 @@ def store(self, istream): if istream.binsha is not None: # copy as much as possible, the actual uncompressed item size might # be smaller than the compressed version - stream_copy(istream.read, writer.write, MAXSIZE, self.stream_chunk_size) + stream_copy(istream.read, writer.write, sys.maxsize, self.stream_chunk_size) else: # write object with header, we have to make a new one write_object(istream.type, istream.size, istream.read, writer.write, diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py index 6909c53e0..c4264748d 100644 --- a/gitdb/utils/compat.py +++ b/gitdb/utils/compat.py @@ -1,8 +1,3 @@ import sys PY3 = sys.version_info[0] == 3 - -try: - MAXSIZE = sys.maxint -except AttributeError: - MAXSIZE = sys.maxsize From c41268071bba593cecd420c400603e094f40a6dc Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 08:01:00 -0600 Subject: [PATCH 0382/1392] Remove and replace encoding.string_types --- gitdb/utils/encoding.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 4d270af9d..156f01677 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -1,10 +1,8 @@ from gitdb.utils import compat if compat.PY3: - string_types = (str, ) text_type = str else: - string_types = (basestring, ) text_type = unicode @@ -12,7 +10,7 @@ def force_bytes(data, encoding="ascii"): if isinstance(data, bytes): return data - if isinstance(data, string_types): + if isinstance(data, str): return data.encode(encoding) return data From 77dc809542d15c40dbe60ff55cd830082c3ad904 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 08:02:08 -0600 Subject: [PATCH 0383/1392] Remove and replace encoding.text_type --- gitdb/utils/encoding.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/gitdb/utils/encoding.py b/gitdb/utils/encoding.py index 156f01677..25ebeadd7 100644 --- a/gitdb/utils/encoding.py +++ b/gitdb/utils/encoding.py @@ -1,11 +1,3 @@ -from gitdb.utils import compat - -if compat.PY3: - text_type = str -else: - text_type = unicode - - def force_bytes(data, encoding="ascii"): if isinstance(data, bytes): return data @@ -17,13 +9,10 @@ def force_bytes(data, encoding="ascii"): def force_text(data, encoding="utf-8"): - if isinstance(data, text_type): + if isinstance(data, str): return data if isinstance(data, bytes): return data.decode(encoding) - if compat.PY3: - return text_type(data, encoding) - else: - return text_type(data) + return str(data, encoding) From db9a65e3b0eceb9c52359273afb3313860e5e322 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 16 Feb 2020 08:11:54 -0600 Subject: [PATCH 0384/1392] Remove compat.PY3 --- gitdb/fun.py | 211 ++++++++++++++---------------------------- gitdb/utils/compat.py | 3 - 2 files changed, 67 insertions(+), 147 deletions(-) delete mode 100644 gitdb/utils/compat.py diff --git a/gitdb/fun.py b/gitdb/fun.py index 92b8b1c70..98465975d 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -16,7 +16,6 @@ from gitdb.const import NULL_BYTE, BYTE_SPACE from gitdb.utils.encoding import force_text -from gitdb.utils.compat import PY3 from gitdb.typ import ( str_blob_type, str_commit_type, @@ -424,20 +423,12 @@ def pack_object_header_info(data): type_id = (c >> 4) & 7 # numeric type size = c & 15 # starting size s = 4 # starting bit-shift size - if PY3: - while c & 0x80: - c = byte_ord(data[i]) - i += 1 - size += (c & 0x7f) << s - s += 7 - # END character loop - else: - while c & 0x80: - c = ord(data[i]) - i += 1 - size += (c & 0x7f) << s - s += 7 - # END character loop + while c & 0x80: + c = byte_ord(data[i]) + i += 1 + size += (c & 0x7f) << s + s += 7 + # END character loop # end performance at expense of maintenance ... return (type_id, size, i) @@ -450,28 +441,16 @@ def create_pack_object_header(obj_type, obj_size): :param obj_type: pack type_id of the object :param obj_size: uncompressed size in bytes of the following object stream""" c = 0 # 1 byte - if PY3: - hdr = bytearray() # output string - - c = (obj_type << 4) | (obj_size & 0xf) - obj_size >>= 4 - while obj_size: - hdr.append(c | 0x80) - c = obj_size & 0x7f - obj_size >>= 7 - # END until size is consumed - hdr.append(c) - else: - hdr = bytes() # output string - - c = (obj_type << 4) | (obj_size & 0xf) - obj_size >>= 4 - while obj_size: - hdr += chr(c | 0x80) - c = obj_size & 0x7f - obj_size >>= 7 - # END until size is consumed - hdr += chr(c) + hdr = bytearray() # output string + + c = (obj_type << 4) | (obj_size & 0xf) + obj_size >>= 4 + while obj_size: + hdr.append(c | 0x80) + c = obj_size & 0x7f + obj_size >>= 7 + # END until size is consumed + hdr.append(c) # end handle interpreter return hdr @@ -484,26 +463,15 @@ def msb_size(data, offset=0): i = 0 l = len(data) hit_msb = False - if PY3: - while i < l: - c = data[i + offset] - size |= (c & 0x7f) << i * 7 - i += 1 - if not c & 0x80: - hit_msb = True - break - # END check msb bit - # END while in range - else: - while i < l: - c = ord(data[i + offset]) - size |= (c & 0x7f) << i * 7 - i += 1 - if not c & 0x80: - hit_msb = True - break - # END check msb bit - # END while in range + while i < l: + c = data[i + offset] + size |= (c & 0x7f) << i * 7 + i += 1 + if not c & 0x80: + hit_msb = True + break + # END check msb bit + # END while in range # end performance ... if not hit_msb: raise AssertionError("Could not find terminating MSB byte in data stream") @@ -663,93 +631,48 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): **Note:** transcribed to python from the similar routine in patch-delta.c""" i = 0 db = delta_buf - if PY3: - while i < delta_buf_size: - c = db[i] - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = db[i] - i += 1 - if (c & 0x02): - cp_off |= (db[i] << 8) - i += 1 - if (c & 0x04): - cp_off |= (db[i] << 16) - i += 1 - if (c & 0x08): - cp_off |= (db[i] << 24) - i += 1 - if (c & 0x10): - cp_size = db[i] - i += 1 - if (c & 0x20): - cp_size |= (db[i] << 8) - i += 1 - if (c & 0x40): - cp_size |= (db[i] << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > src_buf_size): - break - write(src_buf[cp_off:cp_off + cp_size]) - elif c: - write(db[i:i + c]) - i += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data - else: - while i < delta_buf_size: - c = ord(db[i]) - i += 1 - if c & 0x80: - cp_off, cp_size = 0, 0 - if (c & 0x01): - cp_off = ord(db[i]) - i += 1 - if (c & 0x02): - cp_off |= (ord(db[i]) << 8) - i += 1 - if (c & 0x04): - cp_off |= (ord(db[i]) << 16) - i += 1 - if (c & 0x08): - cp_off |= (ord(db[i]) << 24) - i += 1 - if (c & 0x10): - cp_size = ord(db[i]) - i += 1 - if (c & 0x20): - cp_size |= (ord(db[i]) << 8) - i += 1 - if (c & 0x40): - cp_size |= (ord(db[i]) << 16) - i += 1 - - if not cp_size: - cp_size = 0x10000 - - rbound = cp_off + cp_size - if (rbound < cp_size or - rbound > src_buf_size): - break - write(src_buf[cp_off:cp_off + cp_size]) - elif c: - write(db[i:i + c]) - i += c - else: - raise ValueError("unexpected delta opcode 0") - # END handle command byte - # END while processing delta data - # end save byte_ord call and prevent performance regression in py2 + while i < delta_buf_size: + c = db[i] + i += 1 + if c & 0x80: + cp_off, cp_size = 0, 0 + if (c & 0x01): + cp_off = db[i] + i += 1 + if (c & 0x02): + cp_off |= (db[i] << 8) + i += 1 + if (c & 0x04): + cp_off |= (db[i] << 16) + i += 1 + if (c & 0x08): + cp_off |= (db[i] << 24) + i += 1 + if (c & 0x10): + cp_size = db[i] + i += 1 + if (c & 0x20): + cp_size |= (db[i] << 8) + i += 1 + if (c & 0x40): + cp_size |= (db[i] << 16) + i += 1 + + if not cp_size: + cp_size = 0x10000 + + rbound = cp_off + cp_size + if (rbound < cp_size or + rbound > src_buf_size): + break + write(src_buf[cp_off:cp_off + cp_size]) + elif c: + write(db[i:i + c]) + i += c + else: + raise ValueError("unexpected delta opcode 0") + # END handle command byte + # END while processing delta data # yes, lets use the exact same error message that git uses :) assert i == delta_buf_size, "delta replay has gone wild" diff --git a/gitdb/utils/compat.py b/gitdb/utils/compat.py deleted file mode 100644 index c4264748d..000000000 --- a/gitdb/utils/compat.py +++ /dev/null @@ -1,3 +0,0 @@ -import sys - -PY3 = sys.version_info[0] == 3 From 5dd0f302f101e66d9d70a3b17ce0f379b4db214b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 17 Feb 2020 09:15:48 +0800 Subject: [PATCH 0385/1392] bump version to 3.0.1 --- doc/source/changes.rst | 6 ++++++ gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 22deb6db3..9d3b2dca2 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +***** +3.0.1 +***** + +* removed all python2 compatibility shims, GitDB now is a Python 3 program. + ***** 0.6.1 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index c9c827970..6fa31e470 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 0) +version_info = (3, 0, 1) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index a66267e48..12cebcd66 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 0) +version_info = (3, 0, 1) __version__ = '.'.join(str(i) for i in version_info) setup( From f356e12766480852d0e30ae7b786cdf5f24d8cea Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 17 Feb 2020 11:16:43 +0800 Subject: [PATCH 0386/1392] Now with PR: 3.0.2 --- doc/source/changes.rst | 2 +- gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 9d3b2dca2..aa7a8901c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -3,7 +3,7 @@ Changelog ######### ***** -3.0.1 +3.0.2 ***** * removed all python2 compatibility shims, GitDB now is a Python 3 program. diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 6fa31e470..5a52cf205 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 1) +version_info = (3, 0, 2) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 12cebcd66..a66b229f3 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 1) +version_info = (3, 0, 2) __version__ = '.'.join(str(i) for i in version_info) setup( From 02de02cbd938015ff6ba3e668da4e641fdd74c4a Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 22 Feb 2020 16:25:56 -0600 Subject: [PATCH 0387/1392] Restrict smmap2 version to <3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a66b229f3..49cbf3650 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), license="BSD License", zip_safe=False, - install_requires=['smmap2 >= 2.0.0'], + install_requires=['smmap2>=2,<3'], long_description="""GitDB is a pure-Python git object database""", python_requires='>=3.4', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers From 139811a89279482c4df9cddb7d7e69d2e2c36c47 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 22 Feb 2020 16:26:11 -0600 Subject: [PATCH 0388/1392] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ed4898ec2..92b887293 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -smmap>=0.8.3 +smmap2>=2,<3 From 09aa35b62ed341124a7b4757acf35b849a7a39ad Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 22 Feb 2020 18:15:58 -0600 Subject: [PATCH 0389/1392] Improve changelog for v3.0.2 --- doc/source/changes.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index aa7a8901c..e4d4ebc14 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -6,7 +6,8 @@ Changelog 3.0.2 ***** -* removed all python2 compatibility shims, GitDB now is a Python 3 program. +* Removed Python 2 compatibility shims + (`#56 `_) ***** 0.6.1 From 541472db1dcae538875f95216f1655652aa30181 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 23 Feb 2020 09:31:01 +0800 Subject: [PATCH 0390/1392] Change package name to 'smmap'; bump version to 3.0.0 https://github.com/gitpython-developers/smmap/issues/44 --- README.md | 2 +- setup.py | 2 +- smmap/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c7a38173c..175bc3445 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Issues can be filed on github: A link to the pypi page related to this repository: -* https://pypi.org/project/smmap2/ +* https://pypi.org/project/smmap/ ## License Information diff --git a/setup.py b/setup.py index 9136339ae..ea02f457b 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ long_description = "See https://github.com/gitpython-developers/smmap" setup( - name="smmap2", + name="smmap", version=smmap.__version__, description="A pure Python implementation of a sliding window memory map manager", author=smmap.__author__, diff --git a/smmap/__init__.py b/smmap/__init__.py index 1728b832f..2bdf05543 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (2, 0, 5) +version_info = (3, 0, 0) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 1b92f4f4a4ff2bd1057b2b3fcf472e78c3d7abc3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 23 Feb 2020 09:33:52 +0800 Subject: [PATCH 0391/1392] Update makefile to sign with new signature --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 061588245..328c836b9 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ release: clean @test "$$(git rev-parse head)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" make force_release -force_release: clean +force_release:: clean git push --tags - python setup.py sdist bdist_wheel - twine upload -s -i byronimo@gmail.com dist/* + python3 setup.py sdist bdist_wheel + twine upload -s -i 763629FEC8788FC35128B5F6EE029D1E5EB40300 dist/* From 74aef73b0dd1f97b89f95f67500ae9c4c405ff15 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 23 Feb 2020 00:00:54 -0600 Subject: [PATCH 0392/1392] v3.0.3.post1 --- doc/source/changes.rst | 15 +++++++++++++++ gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index e4d4ebc14..ac4b477d4 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,21 @@ Changelog ######### +*********** +3.0.3.post1 +*********** + +* Fixed changelogs for v3.0.2 and v3.0.3 + +***** +3.0.3 +***** + +* Changed ``force_bytes`` to use UTF-8 encoding by default + (`#49 `_) +* Restricted smmap2 version requirement to < 3 +* Updated requirements.txt + ***** 3.0.2 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 5a52cf205..b6a0341c8 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 2) +version_info = (3, 0, 3, "post1") __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 49cbf3650..b79dcb29f 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 2) +version_info = (3, 0, 3, "post1") __version__ = '.'.join(str(i) for i in version_info) setup( From 72698cc333654430a2cc8bd103dc5994d95b3f1b Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 23 Feb 2020 08:27:32 -0600 Subject: [PATCH 0393/1392] Remove carriage returns in setup.py long_description Mitigates https://github.com/pypa/setuptools/issues/1440 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6b71e6795..f0c67ad2d 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ import smmap if os.path.exists("README.md"): - long_description = codecs.open('README.md', "r", "utf-8").read() + long_description = codecs.open('README.md', "r", "utf-8").read().replace('\r\n', '\n') else: long_description = "See https://github.com/gitpython-developers/smmap" From d076f665dae16bd03eeb9df862860d54968eda84 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 23 Feb 2020 08:32:24 -0600 Subject: [PATCH 0394/1392] v3.0.1 --- doc/source/changes.rst | 8 ++++++++ smmap/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index f99e85fb7..e9c2662ef 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,14 @@ Changelog ######### +****** +v3.0.1 +****** +- Switched back to the smmap package name on PyPI and fixed the smmap2 mirror package + (`#44 `_) +- Fixed setup.py ``long_description`` rendering + (`#40 `_) + ********** v0.9.0 ********** diff --git a/smmap/__init__.py b/smmap/__init__.py index 2bdf05543..40861f5be 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (3, 0, 0) +version_info = (3, 0, 1) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 253dfe7092f83229d9e99059e7c51f678a557fd2 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 23 Feb 2020 09:13:36 -0600 Subject: [PATCH 0395/1392] v4.0.1 --- doc/source/changes.rst | 10 ++++++++++ gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- requirements.txt | 2 +- setup.py | 6 +++--- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index ac4b477d4..236b9543e 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,16 @@ Changelog ######### +***** +4.0.1 +***** + +* Switched back to the gitdb package name on PyPI and fixed the gitdb2 mirror package + (`#59 `_) +* Switched back to require smmap package and fixed version requirement to >= 3.0.1, < 4 + (`#59 `_) +* Updated smmap submodule + *********** 3.0.3.post1 *********** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index b6a0341c8..5d68ccbe9 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 3, "post1") +version_info = (4, 0, 1) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index a0060cfdc..d076f665d 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit a0060cfdc9166bb0b3104e8015faf0689aa6daf1 +Subproject commit d076f665dae16bd03eeb9df862860d54968eda84 diff --git a/requirements.txt b/requirements.txt index 92b887293..b6ccf50fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -smmap2>=2,<3 +smmap>=3.0.1,<4 diff --git a/setup.py b/setup.py index b79dcb29f..7617583eb 100755 --- a/setup.py +++ b/setup.py @@ -7,11 +7,11 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (3, 0, 3, "post1") +version_info = (4, 0, 1) __version__ = '.'.join(str(i) for i in version_info) setup( - name="gitdb2", + name="gitdb", version=__version__, description="Git Object Database", author=__author__, @@ -20,7 +20,7 @@ packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), license="BSD License", zip_safe=False, - install_requires=['smmap2>=2,<3'], + install_requires=['smmap>=3.0.1,<4'], long_description="""GitDB is a pure-Python git object database""", python_requires='>=3.4', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers From 02d83a8bad286eba61de54616f1df183825800de Mon Sep 17 00:00:00 2001 From: Nicusor97 Date: Mon, 24 Feb 2020 17:56:18 +0200 Subject: [PATCH 0396/1392] Remove setup.cfg because the gitdb dropped Python 2 support --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 3c6e79cf3..000000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal=1 From 9b0309553be36cd188b3fd7d2ab95e7497a5e80d Mon Sep 17 00:00:00 2001 From: Harmon Date: Mon, 24 Feb 2020 10:45:10 -0600 Subject: [PATCH 0397/1392] v4.0.2 --- doc/source/changes.rst | 7 +++++++ gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 236b9543e..1e91121da 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,13 @@ Changelog ######### +***** +4.0.2 +***** + +* Updated to release as Pure Python Wheel rather than Universal Wheel + (`#62 `_) + ***** 4.0.1 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 5d68ccbe9..df1f242d4 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -29,7 +29,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 1) +version_info = (4, 0, 2) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 7617583eb..d1100991a 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 1) +version_info = (4, 0, 2) __version__ = '.'.join(str(i) for i in version_info) setup( From 77b39e915ef32ca83fb61276128e41e5d7886dc2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 13:57:34 +0800 Subject: [PATCH 0398/1392] Create pythonpackage.yml --- .github/workflows/pythonpackage.yml | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/pythonpackage.yml diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 000000000..4c67fbf5b --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,44 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1000 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with nose + run: | + pip install nose + ulimit -n 48 + ulimit -n + nosetests -v From cc21afae16d00291ea97cc1f159a4c268b94a198 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:08:42 +0800 Subject: [PATCH 0399/1392] Replace travis with github actions --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 9febff0ed..05511a716 100644 --- a/README.rst +++ b/README.rst @@ -50,8 +50,8 @@ Run the tests with DEVELOPMENT =========== -.. image:: https://travis-ci.org/gitpython-developers/gitdb.svg?branch=master - :target: https://travis-ci.org/gitpython-developers/gitdb +.. image:: https://github.com/gitpython-developers/gitdb/workflows/Python%20package/badge.svg + :target: https://github.com/gitpython-developers/gitdb/actions .. image:: https://ci.appveyor.com/api/projects/status/2qa4km4ln7bfv76r/branch/master?svg=true&passingText=windows%20OK&failingText=windows%20failed :target: https://ci.appveyor.com/project/ankostis/gitpython/branch/master) .. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png From e6cc702bf8fdd5e742ed77d607e7b697d27bcfef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:09:05 +0800 Subject: [PATCH 0400/1392] don't try to use __file__ when using pyoxidizer --- gitdb/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index df1f242d4..2b06b24bb 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -13,7 +13,8 @@ def _init_externals(): """Initialize external projects by putting them into the path""" for module in ('smmap',): - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) + if 'PYOXIDIZER' not in os.environ: + sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) try: __import__(module) From aa1275255741ce55722a72c932c45fc43e501d58 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:16:01 +0800 Subject: [PATCH 0401/1392] Add github actions --- .github/workflows/pythonpackage.yml | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/pythonpackage.yml diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 000000000..4c67fbf5b --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,44 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1000 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with nose + run: | + pip install nose + ulimit -n 48 + ulimit -n + nosetests -v From b22f159593594111c513daddf3ad55c5bb0ba7f9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:19:26 +0800 Subject: [PATCH 0402/1392] Fix github actions - no dependencies need to be installed --- .github/workflows/pythonpackage.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4c67fbf5b..4f06a18d9 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -28,7 +28,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt - name: Lint with flake8 run: | pip install flake8 From 2ceedc91958dc080b31e426150610eef52774f5f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:21:10 +0800 Subject: [PATCH 0403/1392] Remove basestring reference, no py2 support --- smmap/util.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/smmap/util.py b/smmap/util.py index defef1f32..13604c581 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -23,11 +23,7 @@ def buffer(obj, offset, size): def string_types(): - if sys.version_info[0] >= 3: - return str - else: - return basestring - + return str def align_to_mmap(num, round_up): """ From ebec89621ed48b5cba97b695fda28389fafaa0a1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:23:23 +0800 Subject: [PATCH 0404/1392] Update badges to represent reality --- .appveyor.yml | 1 + .travis.yml | 1 + README.md | 4 +--- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 2daadaa4d..2090a0569 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,3 +1,4 @@ +# NOT USED, just for reference. See github actions for CI configuration # CI on Windows via appveyor environment: diff --git a/.travis.yml b/.travis.yml index 9cccb75cf..50e5b97e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +# NOT USED, just for reference. See github actions for CI configuration language: python python: - 2.7 diff --git a/README.md b/README.md index 175bc3445..780168e31 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,7 @@ Although memory maps have many advantages, they represent a very limited system ## Overview -[![Build Status](https://travis-ci.org/gitpython-developers/smmap.svg?branch=master)](https://travis-ci.org/gitpython-developers/smmap) -[![Build status](https://ci.appveyor.com/api/projects/status/kuws846av5lvmugo?svg=true&passingText=windows%20OK&failingText=windows%20failed)](https://ci.appveyor.com/project/Byron/smmap) -[![Coverage Status](https://coveralls.io/repos/gitpython-developers/smmap/badge.png)](https://coveralls.io/r/gitpython-developers/smmap) +![Python package](https://github.com/gitpython-developers/smmap/workflows/Python%20package/badge.svg) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/pr)](http://www.issuestats.com/github/gitpython-developers/smmap) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/issue)](http://www.issuestats.com/github/gitpython-developers/smmap) From f4d7a58b4d96200cd057a38a0758d3c84901f57e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:25:21 +0800 Subject: [PATCH 0405/1392] bump patch level --- doc/source/changes.rst | 6 ++++++ smmap/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index e9c2662ef..e5b0e79ce 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +****** +v3.0.2 +****** + +- signed release + ****** v3.0.1 ****** diff --git a/smmap/__init__.py b/smmap/__init__.py index 40861f5be..72df0cf3a 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (3, 0, 1) +version_info = (3, 0, 2) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 57d3f7544820f9fd4202f4ebd0f198c43c8e575d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:35:01 +0800 Subject: [PATCH 0406/1392] bump patch level; mark travis-ci as unused --- .travis.yml | 1 + gitdb/ext/smmap | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 17d63804d..b980d36e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +# NOT USED, just for reference. See github actions for CI configuration language: python python: - "3.4" diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index d076f665d..f4d7a58b4 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit d076f665dae16bd03eeb9df862860d54968eda84 +Subproject commit f4d7a58b4d96200cd057a38a0758d3c84901f57e diff --git a/setup.py b/setup.py index d1100991a..e8f8d68c4 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 2) +version_info = (4, 0, 3) __version__ = '.'.join(str(i) for i in version_info) setup( From 0002408bc03033ed90041d04c4547343a1505537 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:37:37 +0800 Subject: [PATCH 0407/1392] update changelog; remove sublime text (editor) configuration --- doc/source/changes.rst | 6 ++++ etc/sublime-text/gitdb.sublime-project | 44 -------------------------- 2 files changed, 6 insertions(+), 44 deletions(-) delete mode 100644 etc/sublime-text/gitdb.sublime-project diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 1e91121da..3c116c309 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +***** +4.0.3 +***** + +* Support for PyOxidizer + ***** 4.0.2 ***** diff --git a/etc/sublime-text/gitdb.sublime-project b/etc/sublime-text/gitdb.sublime-project deleted file mode 100644 index d0e2e5132..000000000 --- a/etc/sublime-text/gitdb.sublime-project +++ /dev/null @@ -1,44 +0,0 @@ -{ - "folders": - [ - // GITDB - //////// - { - "follow_symlinks": true, - "path": "../..", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", - "gitdb/ext", - "dist", - "doc/build", - ".tox" - ] - }, - // SMMAP - //////// - { - "follow_symlinks": true, - "path": "../../gitdb/ext/smmap", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", - "dist", - "doc/build", - ".tox" - ] - }, - ] -} From b8ab128b7ff8dc951ac42c1fd57d61c26864680a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:40:07 +0800 Subject: [PATCH 0408/1392] =?UTF-8?q?Bump=20patch=20again;=20=E2=80=A6uplo?= =?UTF-8?q?ad=20error=20due=20to=20state=20on=20disk,=20yikes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- doc/source/changes.rst | 2 +- gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 3c116c309..69a6dba98 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -3,7 +3,7 @@ Changelog ######### ***** -4.0.3 +4.0.4 ***** * Support for PyOxidizer diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 2b06b24bb..5f56773d0 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -30,7 +30,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 2) +version_info = (4, 0, 4) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index e8f8d68c4..48d641a8c 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 3) +version_info = (4, 0, 4) __version__ = '.'.join(str(i) for i in version_info) setup( From 4f08b0a2bb6715b1f143782c64e18dc33bb77487 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:44:35 +0800 Subject: [PATCH 0409/1392] Remove windows build badge [skip CI] --- README.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.rst b/README.rst index 05511a716..13dad89fc 100644 --- a/README.rst +++ b/README.rst @@ -52,8 +52,6 @@ DEVELOPMENT .. image:: https://github.com/gitpython-developers/gitdb/workflows/Python%20package/badge.svg :target: https://github.com/gitpython-developers/gitdb/actions -.. image:: https://ci.appveyor.com/api/projects/status/2qa4km4ln7bfv76r/branch/master?svg=true&passingText=windows%20OK&failingText=windows%20failed - :target: https://ci.appveyor.com/project/ankostis/gitpython/branch/master) .. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png :target: https://coveralls.io/r/gitpython-developers/gitdb From 919695d4e4101237a8d2c4fb65807a42b7ff63d4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:45:44 +0800 Subject: [PATCH 0410/1392] Also remove coveralls [skip CI] --- README.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.rst b/README.rst index 13dad89fc..44b3eddf8 100644 --- a/README.rst +++ b/README.rst @@ -52,8 +52,6 @@ DEVELOPMENT .. image:: https://github.com/gitpython-developers/gitdb/workflows/Python%20package/badge.svg :target: https://github.com/gitpython-developers/gitdb/actions -.. image:: https://coveralls.io/repos/gitpython-developers/gitdb/badge.png - :target: https://coveralls.io/r/gitpython-developers/gitdb The library is considered mature, and not under active development. It's primary (known) use is in git-python. From 5a42c622639b12977f8735043f059704c869427d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:46:28 +0800 Subject: [PATCH 0411/1392] Remove issuestats, which doesn't exist anymore [skip CI] --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 780168e31..fcb9217f2 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,6 @@ Although memory maps have many advantages, they represent a very limited system ## Overview ![Python package](https://github.com/gitpython-developers/smmap/workflows/Python%20package/badge.svg) -[![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/pr)](http://www.issuestats.com/github/gitpython-developers/smmap) -[![Issue Stats](http://www.issuestats.com/github/gitpython-developers/smmap/badge/issue)](http://www.issuestats.com/github/gitpython-developers/smmap) Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. From 24668c4e2463aad775edb9d23732d5d11d0e7754 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 19:16:11 +0800 Subject: [PATCH 0412/1392] Change required key to 2CF*, which seems to be the only good one --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 328c836b9..b867ad70c 100644 --- a/Makefile +++ b/Makefile @@ -46,4 +46,4 @@ release: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i 763629FEC8788FC35128B5F6EE029D1E5EB40300 dist/* + twine upload -s -i 2CF6E0B51AAF73F09B1C21174D1DA68C88710E60 dist/* From 09f96f289dbb674e64668bcb0a088aae9dff2a29 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 19:17:40 +0800 Subject: [PATCH 0413/1392] re-release with different signing key --- doc/source/changes.rst | 2 +- smmap/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index e5b0e79ce..f859b6076 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -3,7 +3,7 @@ Changelog ######### ****** -v3.0.2 +v3.0.3 ****** - signed release diff --git a/smmap/__init__.py b/smmap/__init__.py index 72df0cf3a..e7e43d8cd 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (3, 0, 2) +version_info = (3, 0, 3) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From a5d3d7e7ec4e2b52c93509bdf35999d66f91e06d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 19:41:43 +0800 Subject: [PATCH 0414/1392] change package signing key back to what it was --- Makefile | 2 +- gitdb/ext/smmap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 82c0a3bc8..a5098b754 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ release:: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i 763629FEC8788FC35128B5F6EE029D1E5EB40300 dist/* + twine upload -s -i 2CF6E0B51AAF73F09B1C21174D1DA68C88710E60 dist/* doc:: make -C doc/ html diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index f4d7a58b4..09f96f289 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit f4d7a58b4d96200cd057a38a0758d3c84901f57e +Subproject commit 09f96f289dbb674e64668bcb0a088aae9dff2a29 From 5e5f940dff80beaa3eedf9342ef502f5e630d5ed Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 May 2020 11:37:02 +0800 Subject: [PATCH 0415/1392] Bump patch level (to create a new correctly signed release) --- doc/source/changes.rst | 6 ++++++ smmap/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index f859b6076..f8ddb1da0 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +****** +v3.0.4 +****** + +- signed release (with correct key this time) + ****** v3.0.3 ****** diff --git a/smmap/__init__.py b/smmap/__init__.py index e7e43d8cd..a6671e104 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (3, 0, 3) +version_info = (3, 0, 4) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 163f2649e5a5f7b8ba03fc1714bf4693b1a015d0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 5 May 2020 11:45:15 +0800 Subject: [PATCH 0416/1392] Bump patch level for creating a new properly signed release --- doc/source/changes.rst | 6 ++++++ gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 69a6dba98..05748a08d 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +***** +4.0.5 +***** + +* Re-release of 4.0.4, with known signature + ***** 4.0.4 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 5f56773d0..813fe5af6 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -30,7 +30,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 4) +version_info = (4, 0, 5) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 09f96f289..5e5f940df 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 09f96f289dbb674e64668bcb0a088aae9dff2a29 +Subproject commit 5e5f940dff80beaa3eedf9342ef502f5e630d5ed diff --git a/setup.py b/setup.py index 48d641a8c..facdf3d73 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 4) +version_info = (4, 0, 5) __version__ = '.'.join(str(i) for i in version_info) setup( From e5410b4166d177f90901db4986753787d34bc48f Mon Sep 17 00:00:00 2001 From: Ram Rachum Date: Fri, 12 Jun 2020 13:52:38 +0300 Subject: [PATCH 0417/1392] Fix exception causes in loose.py --- gitdb/db/loose.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 7bf92dacf..a63a2ef33 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -138,12 +138,12 @@ def _map_loose_object(self, sha): # try again without noatime try: return file_contents_ro_filepath(db_path) - except OSError: - raise BadObject(sha) + except OSError as new_e: + raise BadObject(sha) from new_e # didn't work because of our flag, don't try it again self._fd_open_flags = 0 else: - raise BadObject(sha) + raise BadObject(sha) from e # END handle error # END exception handling From 112252cef0d418fd070671e64b18558c2f2cf2f1 Mon Sep 17 00:00:00 2001 From: Ram Rachum Date: Sun, 14 Jun 2020 14:35:48 +0300 Subject: [PATCH 0418/1392] Fix exception causes all over the codebase --- gitdb/__init__.py | 4 ++-- gitdb/db/mem.py | 4 ++-- gitdb/util.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 5f56773d0..31e4d4583 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -18,8 +18,8 @@ def _init_externals(): try: __import__(module) - except ImportError: - raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) + except ImportError as e: + raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) from e # END verify import # END handel imports diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 871133468..5b242e46e 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -74,8 +74,8 @@ def stream(self, sha): # rewind stream for the next one to read ostream.stream.seek(0) return ostream - except KeyError: - raise BadObject(sha) + except KeyError as e: + raise BadObject(sha) from e # END exception handling def size(self): diff --git a/gitdb/util.py b/gitdb/util.py index d680f9766..c4cafecc6 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -326,8 +326,8 @@ def open(self, write=False, stream=False): else: self._fd = fd # END handle file descriptor - except OSError: - raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) + except OSError as e: + raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) from e # END handle lock retrieval # open actual file if required From c96c755fa30277fbaadf79603a0b4fa1054ce2cb Mon Sep 17 00:00:00 2001 From: Ram Rachum Date: Mon, 15 Jun 2020 10:48:01 +0300 Subject: [PATCH 0419/1392] Add Ram Rachum to AUTHORS --- AUTHORS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AUTHORS b/AUTHORS index 490baad8e..6c7e9b997 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1 +1,4 @@ Creator: Sebastian Thiel + +Contributors: + - Ram Rachum (@cool-RR) From 56cbfa002a44dce471b58ca3e6680d89ba9ec8a0 Mon Sep 17 00:00:00 2001 From: Harmon Date: Mon, 18 Jan 2021 13:31:13 -0600 Subject: [PATCH 0420/1392] Fix changelog version number v3.0.3 was skipped --- doc/source/changes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index f8ddb1da0..3169674de 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -9,7 +9,7 @@ v3.0.4 - signed release (with correct key this time) ****** -v3.0.3 +v3.0.2 ****** - signed release From 0b53ddc88a1221a9b933bc53570729c96cb4f09d Mon Sep 17 00:00:00 2001 From: Harmon Date: Mon, 18 Jan 2021 13:44:30 -0600 Subject: [PATCH 0421/1392] Remove Sublime Text project definition --- etc/sublime-text/smmap.sublime-project | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 etc/sublime-text/smmap.sublime-project diff --git a/etc/sublime-text/smmap.sublime-project b/etc/sublime-text/smmap.sublime-project deleted file mode 100644 index 251ebbd28..000000000 --- a/etc/sublime-text/smmap.sublime-project +++ /dev/null @@ -1,21 +0,0 @@ -{ - "folders": - [ - // SMMAP - //////// - { - "follow_symlinks": true, - "path": "../..", - "file_exclude_patterns" : [ - "*.sublime-workspace", - ".git", - ".noseids", - ".coverage" - ], - "folder_exclude_patterns" : [ - ".git", - "cover", - ] - }, - ] -} From e79fe25f16c3a8bfcf315b269412f0927486155d Mon Sep 17 00:00:00 2001 From: Harmon Date: Mon, 18 Jan 2021 13:48:06 -0600 Subject: [PATCH 0422/1392] Remove duplicate sys import --- smmap/buf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index 786775a8b..cdbfad3a8 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -3,8 +3,6 @@ __all__ = ["SlidingWindowMapBuffer"] -import sys - try: bytes except NameError: From 066f038cfc74d6095289b5dc3f8810a242e81ecb Mon Sep 17 00:00:00 2001 From: Harmon Date: Mon, 18 Jan 2021 13:53:34 -0600 Subject: [PATCH 0423/1392] Remove bytes existence check Python < 2.6 is no longer supported --- smmap/buf.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/smmap/buf.py b/smmap/buf.py index cdbfad3a8..18766daf8 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -3,11 +3,6 @@ __all__ = ["SlidingWindowMapBuffer"] -try: - bytes -except NameError: - bytes = str - class SlidingWindowMapBuffer(object): From 93c7ba67f0614416482916741f24829bef06e22f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 11 Apr 2020 14:21:10 +0800 Subject: [PATCH 0424/1392] Revert "Remove basestring reference, no py2 support" This reverts commit 2ceedc91958dc080b31e426150610eef52774f5f. --- smmap/util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/smmap/util.py b/smmap/util.py index 13604c581..defef1f32 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -23,7 +23,11 @@ def buffer(obj, offset, size): def string_types(): - return str + if sys.version_info[0] >= 3: + return str + else: + return basestring + def align_to_mmap(num, round_up): """ From 06c4972c87dcd07bb42071b86c2b0d2aae03e581 Mon Sep 17 00:00:00 2001 From: Harmon Date: Thu, 21 Jan 2021 06:06:50 -0600 Subject: [PATCH 0425/1392] Ignore Flake8 undefined name error for Python 2 support --- smmap/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smmap/util.py b/smmap/util.py index defef1f32..3d7329158 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -26,7 +26,7 @@ def string_types(): if sys.version_info[0] >= 3: return str else: - return basestring + return basestring # noqa: F821 def align_to_mmap(num, round_up): From c661324746a26df609d1ace2106a1c8b916cf802 Mon Sep 17 00:00:00 2001 From: Harmon Date: Thu, 21 Jan 2021 06:12:51 -0600 Subject: [PATCH 0426/1392] Improve changelog for v3.0.2 --- doc/source/changes.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 3169674de..f92db64ad 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -12,7 +12,8 @@ v3.0.4 v3.0.2 ****** -- signed release +- Signed release +- Switched to GitHub Actions for CI ****** v3.0.1 From 37cc3c09c188b65996e170cddce2d151bf682388 Mon Sep 17 00:00:00 2001 From: Harmon Date: Thu, 21 Jan 2021 06:14:03 -0600 Subject: [PATCH 0427/1392] Improve capitalization consistency in changelog for v3.0.4 --- doc/source/changes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index f92db64ad..bc1db5d59 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -6,7 +6,7 @@ Changelog v3.0.4 ****** -- signed release (with correct key this time) +- Signed release (with correct key this time) ****** v3.0.2 From 65f171aadffb2d83124065fc6c193f133936643b Mon Sep 17 00:00:00 2001 From: Harmon Date: Thu, 21 Jan 2021 06:17:49 -0600 Subject: [PATCH 0428/1392] Add changelog for v3.0.5 --- doc/source/changes.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index bc1db5d59..c9665b673 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +****** +v3.0.5 +****** + +- Restored Python 2 support removed in v3.0.2 + ****** v3.0.4 ****** From 119cc417541f400b3533c02d53d3d5f236e87023 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 23 Jan 2021 09:59:05 +0800 Subject: [PATCH 0429/1392] bump patch level --- Makefile | 2 +- doc/source/changes.rst | 2 ++ smmap/__init__.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b867ad70c..593a758ee 100644 --- a/Makefile +++ b/Makefile @@ -46,4 +46,4 @@ release: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i 2CF6E0B51AAF73F09B1C21174D1DA68C88710E60 dist/* + twine upload -s -i 27C50E7F590947D7273A741E85194C08421980C9 dist/* diff --git a/doc/source/changes.rst b/doc/source/changes.rst index c9665b673..041711bae 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -7,6 +7,8 @@ v3.0.5 ****** - Restored Python 2 support removed in v3.0.2 +- changed release signature key to 27C50E7F590947D7273A741E85194C08421980C9. See + https://keybase.io/byronbates for proof of ownership. ****** v3.0.4 diff --git a/smmap/__init__.py b/smmap/__init__.py index a6671e104..abc09d952 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (3, 0, 4) +version_info = (3, 0, 5) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 9c2570bf07db6dfc7d616b5ffaee84cc33b5c0f5 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:01:32 -0600 Subject: [PATCH 0430/1392] Improve changelog for v3.0.5 --- doc/source/changes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 041711bae..bc0b1b63c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -7,8 +7,8 @@ v3.0.5 ****** - Restored Python 2 support removed in v3.0.2 -- changed release signature key to 27C50E7F590947D7273A741E85194C08421980C9. See - https://keybase.io/byronbates for proof of ownership. +- Changed release signature key to 27C50E7F590947D7273A741E85194C08421980C9. + See https://keybase.io/byronbates for proof of ownership. ****** v3.0.4 From 341b27890a0e1353ccdf08b345aee9f5934d1847 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:08:03 -0600 Subject: [PATCH 0431/1392] Remove string_types --- smmap/mman.py | 3 +-- smmap/util.py | 11 ++--------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index 9df69ed29..d7dfe6d62 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -4,7 +4,6 @@ MapRegion, MapRegionList, is_64_bit, - string_types, buffer, ) @@ -227,7 +226,7 @@ def fd(self): **Note:** it is not required to be valid anymore :raise ValueError: if the mapping was not created by a file descriptor""" - if isinstance(self._rlist.path_or_fd(), string_types()): + if isinstance(self._rlist.path_or_fd(), str): raise ValueError("File descriptor queried although mapping was generated from path") # END handle type return self._rlist.path_or_fd() diff --git a/smmap/util.py b/smmap/util.py index 3d7329158..4e713f8ae 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -22,13 +22,6 @@ def buffer(obj, offset, size): # return obj[offset:offset + size] -def string_types(): - if sys.version_info[0] >= 3: - return str - else: - return basestring # noqa: F821 - - def align_to_mmap(num, round_up): """ Align the given integer number to the closest page offset, which usually is 4096 bytes. @@ -146,7 +139,7 @@ def __init__(self, path_or_fd, ofs, size, flags=0): self._size = len(self._mf) finally: - if isinstance(path_or_fd, string_types()): + if isinstance(path_or_fd, str): os.close(fd) # END only close it if we opened it # END close file handle @@ -229,7 +222,7 @@ def path_or_fd(self): def file_size(self): """:return: size of file we manager""" if self._file_size is None: - if isinstance(self._path_or_fd, string_types()): + if isinstance(self._path_or_fd, str): self._file_size = os.stat(self._path_or_fd).st_size else: self._file_size = os.fstat(self._path_or_fd).st_size From da0a610bd9a050239299d37ce40d4c01d5cacc81 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:15:53 -0600 Subject: [PATCH 0432/1392] Remove buffer --- smmap/mman.py | 3 +-- smmap/util.py | 13 +------------ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/smmap/mman.py b/smmap/mman.py index d7dfe6d62..19c3a0223 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -4,7 +4,6 @@ MapRegion, MapRegionList, is_64_bit, - buffer, ) import sys @@ -160,7 +159,7 @@ def buffer(self): **Note:** buffers should not be cached passed the duration of your access as it will prevent resources from being freed even though they might not be accounted for anymore !""" - return buffer(self._region.buffer(), self._ofs, self._size) + return memoryview(self._region.buffer())[self._ofs:self._ofs+self._size] def map(self): """ diff --git a/smmap/util.py b/smmap/util.py index 4e713f8ae..72b2394a3 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -5,22 +5,11 @@ from mmap import mmap, ACCESS_READ from mmap import ALLOCATIONGRANULARITY -__all__ = ["align_to_mmap", "is_64_bit", "buffer", +__all__ = ["align_to_mmap", "is_64_bit", "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities -try: - # Python 2 - buffer = buffer -except NameError: - # Python 3 has no `buffer`; only `memoryview` - def buffer(obj, offset, size): - # Actually, for gitpython this is fastest ... . - return memoryview(obj)[offset:offset+size] - # doing it directly is much faster ! - # return obj[offset:offset + size] - def align_to_mmap(num, round_up): """ From 0e59cc906065c36e2605f9b76d797d5227f84460 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:20:13 -0600 Subject: [PATCH 0433/1392] Update Python requirement to >= 3.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f0c67ad2d..c4a9f4e4f 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + python_requires=">=3.4", classifiers=[ # Picked from # http://pypi.python.org/pypi?:action=list_classifiers From ee01c2e6f661b76ddde94cdfa02176dc573cd95b Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:21:16 -0600 Subject: [PATCH 0434/1392] Update setup classifiers to be only Python 3 --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c4a9f4e4f..d2c6a21d2 100755 --- a/setup.py +++ b/setup.py @@ -45,12 +45,11 @@ "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3 :: Only", ], long_description=long_description, long_description_content_type='text/markdown', From 9478708e6663fabbeb8464d6a943054e8927c2ce Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:25:14 -0600 Subject: [PATCH 0435/1392] Update Python prerequisite in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fcb9217f2..9e6ea427c 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 2.7 or 3.4+ +* Python 3.4+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. From cc7d2b8516a9bc23d7b3ac599395b80461b738cd Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:26:22 -0600 Subject: [PATCH 0436/1392] Remove information about Python < 2.5 in README --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 9e6ea427c..9707e5036 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,6 @@ Smmap wraps an interface around mmap and tracks the mapped files as well as the To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. -The interface also works around the missing offset parameter in python implementations up to python 2.5. - Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. From 3646d133e3be66ce1297560542f43b547d47e318 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:27:08 -0600 Subject: [PATCH 0437/1392] Remove Python 2.7 from tox configuration --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d1f558bc0..8a5ce02ad 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py27, py34, py35, py36 +envlist = flake8, py34, py35, py36 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From d20070534a443543fabfae24e249921821a0f991 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:29:39 -0600 Subject: [PATCH 0438/1392] Update Python prerequisite in documentation --- doc/source/intro.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 8f7cf3c26..63480fc28 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -22,7 +22,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 2.7 or 3.4+ +* Python 3.4+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. From 72d47eb937f22de5c71abe5630af6c1dc999bbe0 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:30:19 -0600 Subject: [PATCH 0439/1392] Remove information about Python < 2.5 in documentation --- doc/source/intro.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 63480fc28..a4c82d295 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -13,8 +13,6 @@ Smmap wraps an interface around mmap and tracks the mapped files as well as the To allow processing large files even on 32 bit systems, it allows only portions of the file to be mapped. Once the user reads beyond the mapped region, smmap will automatically map the next required region, unloading unused regions using a LRU algorithm. -The interface also works around the missing offset parameter in python implementations up to python 2.5. - Although the library can be used most efficiently with its native interface, a Buffer implementation is provided to hide these details behind a simple string-like interface. For performance critical 64 bit applications, a simplified version of memory mapping is provided which always maps the whole file, but still provides the benefit of unloading unused mappings on demand. From 812ad85acd3eda5b1d6dc5aa837d390a72ce1538 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:32:29 -0600 Subject: [PATCH 0440/1392] Replace codecs.open with open --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d2c6a21d2..46eecdd9b 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python import os -import codecs + try: from setuptools import setup, find_packages except ImportError: @@ -11,7 +11,7 @@ import smmap if os.path.exists("README.md"): - long_description = codecs.open('README.md', "r", "utf-8").read().replace('\r\n', '\n') + long_description = open('README.md', "r", encoding="utf-8").read().replace('\r\n', '\n') else: long_description = "See https://github.com/gitpython-developers/smmap" From 960cbc5b01aafe4d0b1706ab236d40d1125e05ac Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:42:41 -0600 Subject: [PATCH 0441/1392] Remove buffer usage from TestTutorial.test_example --- smmap/test/test_tutorial.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index b03db9be2..31c272abb 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -41,12 +41,6 @@ def test_example(self): c.buffer()[1:10] # first 9 bytes c.buffer()[c.size() - 1] # last byte - # its recommended not to create big slices when feeding the buffer - # into consumers (e.g. struct or zlib). - # Instead, either give the buffer directly, or use pythons buffer command. - from smmap.util import buffer - buffer(c.buffer(), 1, 9) # first 9 bytes without copying them - # you can query absolute offsets, and check whether an offset is included # in the cursor's data. assert c.ofs_begin() < c.ofs_end() From 230725f81e0298d110f8b136e1a6a19c9105d961 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:45:48 -0600 Subject: [PATCH 0442/1392] Remove Development Status classifier comments in setup.py --- setup.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/setup.py b/setup.py index 46eecdd9b..755fddafd 100755 --- a/setup.py +++ b/setup.py @@ -28,15 +28,7 @@ zip_safe=True, python_requires=">=3.4", classifiers=[ - # Picked from - # http://pypi.python.org/pypi?:action=list_classifiers - #"Development Status :: 1 - Planning", - #"Development Status :: 2 - Pre-Alpha", - #"Development Status :: 3 - Alpha", - # "Development Status :: 4 - Beta", "Development Status :: 5 - Production/Stable", - #"Development Status :: 6 - Mature", - #"Development Status :: 7 - Inactive", "Environment :: Console", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", From 710fb1994adda7215261d240e6332c2a10bfff6a Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:48:38 -0600 Subject: [PATCH 0443/1392] Remove Travis CI configuration file --- .travis.yml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 50e5b97e1..000000000 --- a/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -# NOT USED, just for reference. See github actions for CI configuration -language: python -python: - - 2.7 - - 3.4 - - 3.5 - - 3.6 -install: - - pip install coveralls -script: - - ulimit -n 48 - - ulimit -n - - nosetests --with-coverage -after_success: - - coveralls From 724540cd82856e850ab9175775db8caa1ea42ec7 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:49:29 -0600 Subject: [PATCH 0444/1392] Remove AppVeyor configuration file --- .appveyor.yml | 50 -------------------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 2090a0569..000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,50 +0,0 @@ -# NOT USED, just for reference. See github actions for CI configuration -# CI on Windows via appveyor -environment: - - matrix: - ## MINGW - # - - PYTHON: "C:\\Python27" - PYTHON_VERSION: "2.7" - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4" - - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5" - - PYTHON: "C:\\Miniconda35-x64" - PYTHON_VERSION: "3.5" - IS_CONDA: "yes" - -install: - - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% - - ## Print configuration for debugging. - # - - | - echo %PATH% - uname -a - where python pip pip2 pip3 pip34 - python --version - python -c "import struct; print(struct.calcsize('P') * 8)" - - - IF "%IS_CONDA%"=="yes" ( - conda info -a & - conda install --yes --quiet pip - ) - - pip install nose wheel coveralls - - ## For commits performed with the default user. - - | - git config --global user.email "travis@ci.com" - git config --global user.name "Travis Runner" - - - pip install -e . - -build: false - -test_script: - - IF "%PYTHON_VERSION%"=="3.5" ( - nosetests -v --with-coverage - ) ELSE ( - nosetests -v - ) From 18b499e213b1fd81c4cf5e5340a5af6302f26bef Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:49:54 -0600 Subject: [PATCH 0445/1392] Remove MemoryManagerError and RegionCollectionError --- smmap/exc.py | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 smmap/exc.py diff --git a/smmap/exc.py b/smmap/exc.py deleted file mode 100644 index 117664502..000000000 --- a/smmap/exc.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Module with system exceptions""" - - -class MemoryManagerError(Exception): - - """Base class for all exceptions thrown by the memory manager""" - - -class RegionCollectionError(MemoryManagerError): - - """Thrown if a memory region could not be collected, or if no region for collection was found""" From fe64cbd88c37e0a5fb52709790b44cba7a5f5fd8 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:52:16 -0600 Subject: [PATCH 0446/1392] Remove Exceptions documentation --- doc/source/api.rst | 8 -------- 1 file changed, 8 deletions(-) diff --git a/doc/source/api.rst b/doc/source/api.rst index cddd268c4..2e2dac41a 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -20,14 +20,6 @@ Buffers :members: :undoc-members: -********** -Exceptions -********** - -.. automodule:: smmap.exc - :members: - :undoc-members: - ********* Utilities ********* From 95431a22d559c17e96ed2ca094247f0d227a544c Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:55:43 -0600 Subject: [PATCH 0447/1392] Add Python 3.9 to GitHub Actions workflow --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4f06a18d9..e070c68f8 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 From 8fda48842fb422025b498cd21644b7a480593cc2 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 21:57:21 -0600 Subject: [PATCH 0448/1392] Add Python 3.7, 3.8, and 3.9 to setup classifiers --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 755fddafd..039f30241 100755 --- a/setup.py +++ b/setup.py @@ -41,6 +41,9 @@ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3 :: Only", ], long_description=long_description, From c64ac58cf768d5416fcc3e306fa72843089313a2 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 22:00:26 -0600 Subject: [PATCH 0449/1392] Drop support for Python 3.4 --- README.md | 2 +- doc/source/intro.rst | 2 +- setup.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9707e5036..412dfd9c0 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 3.4+ +* Python 3.5+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index a4c82d295..e458d53aa 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -20,7 +20,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 3.4+ +* Python 3.5+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/setup.py b/setup.py index 039f30241..0718c15ef 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, - python_requires=">=3.4", + python_requires=">=3.5", classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -38,7 +38,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", From 689ab6fb040e218ca16e4c0aa4944e608649df04 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 22:02:43 -0600 Subject: [PATCH 0450/1392] Add Python 3.7, 3.8, and 3.9 to tox configuration --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8a5ce02ad..9e8dd22b2 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py34, py35, py36 +envlist = flake8, py34, py35, py36, py37, py38, py39 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From ed11471388a3c4dc4756605ec76650e7ad994eb9 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 22:03:00 -0600 Subject: [PATCH 0451/1392] Remove Python 3.4 from tox configuration --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9e8dd22b2..e33f567f2 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py34, py35, py36, py37, py38, py39 +envlist = flake8, py35, py36, py37, py38, py39 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From e762f17a0bc479d52d4f9cda46cec3f98c89e812 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 22:06:54 -0600 Subject: [PATCH 0452/1392] Remove unused pyvers variable in SlidingWindowMapBuffer.__getslice__ --- smmap/buf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/smmap/buf.py b/smmap/buf.py index 18766daf8..af8496eff 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -80,7 +80,6 @@ def __getslice__(self, i, j): ofs = i # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower # in the previous iteration of this code - pyvers = sys.version_info[:2] md = list() while l: c.use_region(ofs, l) From 30e93fee57286afae25c28a97ba65a9770f9a729 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sat, 23 Jan 2021 22:17:37 -0600 Subject: [PATCH 0453/1392] v4.0.0 --- doc/source/changes.rst | 8 ++++++++ smmap/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index bc0b1b63c..ff5ca9918 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,14 @@ Changelog ######### +****** +v4.0.0 +****** + +- Dropped support for Python 2.7 and 3.4 +- Added support for Python 3.7, 3.8, and 3.9 +- Removed unused exc.MemoryManagerError and exc.RegionCollectionError + ****** v3.0.5 ****** diff --git a/smmap/__init__.py b/smmap/__init__.py index abc09d952..45f8abec8 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (3, 0, 5) +version_info = (4, 0, 0) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 447c8a4b0cc6e8fbc4a20a5a6e2c7cfabe05368e Mon Sep 17 00:00:00 2001 From: Tom McClintock Date: Wed, 24 Mar 2021 08:57:44 -0700 Subject: [PATCH 0454/1392] Bumped smmap upper bound --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b6ccf50fa..72957a243 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -smmap>=3.0.1,<4 +smmap>=3.0.1,<5 From aa7228e8dbdc2ee6b6bc385e8bee21245a10e98d Mon Sep 17 00:00:00 2001 From: Harmon Date: Thu, 25 Mar 2021 06:29:35 -0500 Subject: [PATCH 0455/1392] v4.0.6 --- doc/source/changes.rst | 8 ++++++++ gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 05748a08d..6217ffb99 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,14 @@ Changelog ######### +***** +4.0.6 +***** + +* Bumped upper bound for smmap requirement + (`#67 `_, + `#68 `_) + ***** 4.0.5 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 2e127cac4..ea7d1bcc6 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -30,7 +30,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 5) +version_info = (4, 0, 6) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index facdf3d73..071b8c6c1 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 5) +version_info = (4, 0, 6) __version__ = '.'.join(str(i) for i in version_info) setup( From fc11a03b6d94cdf9d5841595caf104c2982934bb Mon Sep 17 00:00:00 2001 From: Harmon Date: Fri, 26 Mar 2021 07:41:26 -0500 Subject: [PATCH 0456/1392] Update smmap upper bound in setup.py Fixes #69 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 071b8c6c1..edbeeffe3 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), license="BSD License", zip_safe=False, - install_requires=['smmap>=3.0.1,<4'], + install_requires=['smmap>=3.0.1,<5'], long_description="""GitDB is a pure-Python git object database""", python_requires='>=3.4', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers From 6b997cd5fd01dd91ecb08d39e5e9736bc1dc9ba5 Mon Sep 17 00:00:00 2001 From: Harmon Date: Fri, 26 Mar 2021 07:43:45 -0500 Subject: [PATCH 0457/1392] v4.0.7 --- doc/source/changes.rst | 7 +++++++ gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 6217ffb99..1a41d537b 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,13 @@ Changelog ######### +***** +4.0.7 +***** + +* Updated upper bound for smmap requirement in setup.py + (`#69 `_) + ***** 4.0.6 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index ea7d1bcc6..bef9696c7 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -30,7 +30,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 6) +version_info = (4, 0, 7) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index edbeeffe3..59817d8b2 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 6) +version_info = (4, 0, 7) __version__ = '.'.join(str(i) for i in version_info) setup( From 03ab3a1d40c04d6a944299c21db61cf9ce30f6bb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 25 Mar 2021 20:46:35 +0800 Subject: [PATCH 0458/1392] change signing key to the one I have --- Makefile | 2 +- gitdb/ext/smmap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a5098b754..82907561b 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ release:: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i 2CF6E0B51AAF73F09B1C21174D1DA68C88710E60 dist/* + twine upload -s -i 27C50E7F590947D7273A741E85194C08421980C9 dist/* doc:: make -C doc/ html diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 5e5f940df..30e93fee5 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 5e5f940dff80beaa3eedf9342ef502f5e630d5ed +Subproject commit 30e93fee57286afae25c28a97ba65a9770f9a729 From dff15cd8ba473776f76e8a3b6359a861e72d74aa Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 19 Aug 2021 12:09:11 +0300 Subject: [PATCH 0459/1392] Replace deprecated unittest aliases --- gitdb/test/db/lib.py | 4 ++-- gitdb/test/db/test_git.py | 2 +- gitdb/test/db/test_loose.py | 2 +- gitdb/test/db/test_pack.py | 2 +- gitdb/test/test_pack.py | 2 +- gitdb/test/test_stream.py | 2 +- gitdb/test/test_util.py | 6 +++--- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index c6f4316cd..3df326b68 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -102,8 +102,8 @@ def _assert_object_writing(self, db): assert ostream.type == str_blob_type assert ostream.size == len(data) else: - self.failUnlessRaises(BadObject, db.info, sha) - self.failUnlessRaises(BadObject, db.stream, sha) + self.assertRaises(BadObject, db.info, sha) + self.assertRaises(BadObject, db.stream, sha) # DIRECT STREAM COPY # our data hase been written in object format to the StringIO diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index acc0f153f..6ecf7d7a8 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -43,7 +43,7 @@ def test_reading(self): assert gdb.partial_to_complete_sha_hex(bin_to_hex(binsha)[:8 - (i % 2)]) == binsha # END for each sha - self.failUnlessRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") + self.assertRaises(BadObject, gdb.partial_to_complete_sha_hex, "0000") @with_rw_directory def test_writing(self, path): diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 9c25a0249..8cc660b8a 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -32,5 +32,5 @@ def test_basics(self, path): assert bin_to_hex(ldb.partial_to_complete_sha_hex(short_sha)) == long_sha # END for each sha - self.failUnlessRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') + self.assertRaises(BadObject, ldb.partial_to_complete_sha_hex, '0000') # raises if no object could be found diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 9694238bd..458d80434 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -84,4 +84,4 @@ def test_writing(self, path): # assert num_ambiguous # non-existing - self.failUnlessRaises(BadObject, pdb.partial_to_complete_sha, b'\0\0', 4) + self.assertRaises(BadObject, pdb.partial_to_complete_sha, b'\0\0', 4) diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 8bf78f0d2..48a185290 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -73,7 +73,7 @@ def _assert_index_file(self, index, version, size): assert index.partial_sha_to_index(sha[:l], l * 2) == oidx # END for each object index in indexfile - self.failUnlessRaises(ValueError, index.partial_sha_to_index, "\0", 2) + self.assertRaises(ValueError, index.partial_sha_to_index, "\0", 2) def _assert_pack_file(self, pack, version, size): assert pack.version() == 2 diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 96268252d..5d4b93db7 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -135,7 +135,7 @@ def test_compressed_writer(self): ostream.close() # its closed already - self.failUnlessRaises(OSError, os.close, fd) + self.assertRaises(OSError, os.close, fd) # read everything back, compare to data we zip fd = os.open(path, os.O_RDONLY | getattr(os, 'O_BINARY', 0)) diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 847bdab5e..3b3165d14 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -40,8 +40,8 @@ def test_lockedfd(self): lockfilepath = lfd._lockfilepath() # cannot end before it was started - self.failUnlessRaises(AssertionError, lfd.rollback) - self.failUnlessRaises(AssertionError, lfd.commit) + self.assertRaises(AssertionError, lfd.rollback) + self.assertRaises(AssertionError, lfd.commit) # open for writing assert not os.path.isfile(lockfilepath) @@ -77,7 +77,7 @@ def test_lockedfd(self): wfdstream = lfd.open(write=True, stream=True) # this time as stream assert os.path.isfile(lockfilepath) # another one fails - self.failUnlessRaises(IOError, olfd.open) + self.assertRaises(IOError, olfd.open) wfdstream.write(new_data.encode("ascii")) lfd.commit() From 1003c6612e0ee8973ba701e317b7308b7d0136aa Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 19 Aug 2021 12:22:22 +0300 Subject: [PATCH 0460/1392] Fix Sphinx warnings --- doc/source/conf.py | 2 +- gitdb/db/base.py | 5 +++++ gitdb/db/mem.py | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 68d9a3fcd..3ab15ab35 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -120,7 +120,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['.static'] +#html_static_path = ['.static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/gitdb/db/base.py b/gitdb/db/base.py index 2d7b9fa8d..f0b8a0574 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -33,6 +33,9 @@ def __contains__(self, sha): #{ Query Interface def has_object(self, sha): """ + Whether the object identified by the given 20 bytes + binary sha is contained in the database + :return: True if the object identified by the given 20 bytes binary sha is contained in the database""" raise NotImplementedError("To be implemented in subclass") @@ -82,6 +85,8 @@ def set_ostream(self, stream): def ostream(self): """ + Return the output stream + :return: overridden output stream this instance will write to, or None if it will write to the default stream""" return self._ostream diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 5b242e46e..212a68fd1 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -92,6 +92,7 @@ def stream_copy(self, sha_iter, odb): """Copy the streams as identified by sha's yielded by sha_iter into the given odb The streams will be copied directly **Note:** the object will only be written if it did not exist in the target db + :return: amount of streams actually copied into odb. If smaller than the amount of input shas, one or more objects did already exist in odb""" count = 0 From 01b6510de861ed19958ecdd445afaccd2d8a7951 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 19 Aug 2021 12:31:04 +0300 Subject: [PATCH 0461/1392] Remove redundant Python 2.6 code --- gitdb/stream.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index b94ef245d..d58d1a63e 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -30,7 +30,6 @@ from gitdb.utils.encoding import force_bytes has_perf_mod = False -PY26 = sys.version_info[:2] < (2, 7) try: from gitdb_speedups._perf import apply_delta as c_apply_delta has_perf_mod = True @@ -295,7 +294,7 @@ def read(self, size=-1): # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the # table in the github issue. This is it ... it was the only way I could make this work everywhere. # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . - if PY26 or ((zlib.ZLIB_VERSION == '1.2.7' or zlib.ZLIB_VERSION == '1.2.5') and not sys.platform == 'darwin'): + if zlib.ZLIB_VERSION in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': unused_datalen = len(self._zip.unconsumed_tail) else: unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) From e4cd296a2101df10e400ec2f1f7ac8b5ac2c37eb Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 19 Aug 2021 12:32:36 +0300 Subject: [PATCH 0462/1392] Remove redundant Python 2 code --- gitdb/db/mem.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 212a68fd1..b2542ff1b 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -82,10 +82,7 @@ def size(self): return len(self._cache) def sha_iter(self): - try: - return self._cache.iterkeys() - except AttributeError: - return self._cache.keys() + return self._cache.keys() #{ Interface def stream_copy(self, sha_iter, odb): From 0df4b09bac21b7d3e50931c01ccf261419f5e1ef Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 19 Aug 2021 12:42:16 +0300 Subject: [PATCH 0463/1392] Support Python 3.7-3.9 --- .github/workflows/pythonpackage.yml | 4 ++-- setup.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4c67fbf5b..7f35b616e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,14 +15,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 with: fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/setup.py b/setup.py index 59817d8b2..522a88f16 100755 --- a/setup.py +++ b/setup.py @@ -38,6 +38,8 @@ "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7" + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ] ) From 3affd88c09635f0cad0f268c5ca22162c1aa0aa8 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 14 Oct 2021 17:19:42 +0300 Subject: [PATCH 0464/1392] Drop support for EOL Python 3.5 --- .github/workflows/pythonpackage.yml | 2 +- README.md | 2 +- doc/source/intro.rst | 2 +- setup.py | 3 +-- tox.ini | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index e070c68f8..3f1455d96 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + python-version: ["3.6", "3.7", "3.8", "3.9"] steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index 412dfd9c0..f083dd004 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 3.5+ +* Python 3.6+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index e458d53aa..3489b04ee 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -20,7 +20,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 3.5+ +* Python 3.6+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/setup.py b/setup.py index 0718c15ef..833379478 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, - python_requires=">=3.5", + python_requires=">=3.6", classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -38,7 +38,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", diff --git a/tox.ini b/tox.ini index e33f567f2..54f328da1 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py35, py36, py37, py38, py39 +envlist = flake8, py36, py37, py38, py39 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From 19772d24cd63a0b5392084c07cb6a4e8a2ac86ae Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 14 Oct 2021 17:06:35 +0300 Subject: [PATCH 0465/1392] Add support for Python 3.10 --- .github/workflows/pythonpackage.yml | 2 +- setup.py | 1 + tox.ini | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3f1455d96..e8726a269 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index 833379478..c25e445ed 100755 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3 :: Only", ], long_description=long_description, diff --git a/tox.ini b/tox.ini index 54f328da1..6587dcedf 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py36, py37, py38, py39 +envlist = flake8, py36, py37, py38, py39, py310 [testenv] commands = nosetests {posargs:--with-coverage --cover-package=smmap} From ddc4b9f412769915b77d857324ad165ef488499b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 14 Oct 2021 17:14:27 +0300 Subject: [PATCH 0466/1392] Switch from abandoned nose to pytest to support Python 3.10 --- .github/workflows/pythonpackage.yml | 8 ++++---- Makefile | 6 +++--- setup.py | 2 -- tox.ini | 6 +++--- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index e8726a269..e6e68ff7c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -22,7 +22,7 @@ jobs: with: fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -35,9 +35,9 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with nose + - name: Test run: | - pip install nose + pip install pytest ulimit -n 48 ulimit -n - nosetests -v + pytest -v diff --git a/Makefile b/Makefile index 593a758ee..9145cabb0 100644 --- a/Makefile +++ b/Makefile @@ -24,11 +24,11 @@ clean-files: clean: clean-files clean-docs test: - nosetests + pytest coverage: - nosetests --with-coverage --cover-package=smmap - + pytest --cov smmap --cov-report xml + build: ./setup.py build diff --git a/setup.py b/setup.py index c25e445ed..56e560a38 100755 --- a/setup.py +++ b/setup.py @@ -47,6 +47,4 @@ ], long_description=long_description, long_description_content_type='text/markdown', - tests_require=('nose', 'nosexcover'), - test_suite='nose.collector' ) diff --git a/tox.ini b/tox.ini index 6587dcedf..c34ab0290 100644 --- a/tox.ini +++ b/tox.ini @@ -7,10 +7,10 @@ envlist = flake8, py36, py37, py38, py39, py310 [testenv] -commands = nosetests {posargs:--with-coverage --cover-package=smmap} +commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} deps = - nose - nosexcover + pytest + pytest-cov [testenv:flake8] commands = flake8 {posargs} From 1bda3b6a5ee6e383e9fd3316caa2d8181e53bf45 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 14 Oct 2021 17:18:23 +0300 Subject: [PATCH 0467/1392] Universal wheels are for code expected to work on both Python 2 and 3 --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 83a51c4b9..38c535ba5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,2 @@ -[bdist_wheel] -universal = 1 - [flake8] exclude = .tox,.venv,build,dist,doc From f0b322afcf6934501bade7776c5331619485a06c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 14 Oct 2021 17:27:17 +0300 Subject: [PATCH 0468/1392] Upgrade Python syntax with pyupgrade --py36-plus --- doc/source/conf.py | 9 ++++----- setup.py | 2 +- smmap/buf.py | 2 +- smmap/mman.py | 6 +++--- smmap/test/lib.py | 2 +- smmap/test/test_buf.py | 2 -- smmap/test/test_mman.py | 2 -- smmap/util.py | 6 +++--- 8 files changed, 13 insertions(+), 18 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 5aa28e2a6..55dfc5c46 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # smmap documentation build configuration file, created by # sphinx-quickstart on Wed Jun 8 15:14:25 2011. @@ -38,8 +37,8 @@ master_doc = 'index' # General information about the project. -project = u'smmap' -copyright = u'2011, Sebastian Thiel' +project = 'smmap' +copyright = '2011, Sebastian Thiel' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -173,8 +172,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'smmap.tex', u'smmap Documentation', - u'Sebastian Thiel', 'manual'), + ('index', 'smmap.tex', 'smmap Documentation', + 'Sebastian Thiel', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/setup.py b/setup.py index 56e560a38..9ab8a7f0c 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ import smmap if os.path.exists("README.md"): - long_description = open('README.md', "r", encoding="utf-8").read().replace('\r\n', '\n') + long_description = open('README.md', encoding="utf-8").read().replace('\r\n', '\n') else: long_description = "See https://github.com/gitpython-developers/smmap" diff --git a/smmap/buf.py b/smmap/buf.py index af8496eff..795e0fd87 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -4,7 +4,7 @@ __all__ = ["SlidingWindowMapBuffer"] -class SlidingWindowMapBuffer(object): +class SlidingWindowMapBuffer: """A buffer like object which allows direct byte-wise object and slicing into memory of a mapped file. The mapping is controlled by the provided cursor. diff --git a/smmap/mman.py b/smmap/mman.py index 19c3a0223..1de7d9e97 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -15,7 +15,7 @@ #}END utilities -class WindowCursor(object): +class WindowCursor: """ Pointer into the mapped region of the memory manager, keeping the map @@ -233,7 +233,7 @@ def fd(self): #} END interface -class StaticWindowMapManager(object): +class StaticWindowMapManager: """Provides a manager which will produce single size cursors that are allowed to always map the whole file. @@ -486,7 +486,7 @@ class SlidingWindowMapManager(StaticWindowMapManager): def __init__(self, window_size=-1, max_memory_size=0, max_open_handles=sys.maxsize): """Adjusts the default window size to -1""" - super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles) + super().__init__(window_size, max_memory_size, max_open_handles) def _obtain_region(self, a, offset, size, flags, is_recursive): # bisect to find an existing region. The c++ implementation cannot diff --git a/smmap/test/lib.py b/smmap/test/lib.py index f86c0c6f1..ca91ee914 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -8,7 +8,7 @@ #{ Utilities -class FileCreator(object): +class FileCreator: """A instance which creates a temporary file with a prefix and a given size and provides this info to the user. diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 3b6009e11..17555afe4 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from .lib import TestBase, FileCreator from smmap.mman import ( diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index 96bc355b7..d88316b8e 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from .lib import TestBase, FileCreator from smmap.mman import ( diff --git a/smmap/util.py b/smmap/util.py index 72b2394a3..cf027afdc 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -34,7 +34,7 @@ def is_64_bit(): #{ Utility Classes -class MapWindow(object): +class MapWindow: """Utility type which is used to snap windows towards each other, and to adjust their size""" __slots__ = ( @@ -80,7 +80,7 @@ def extend_right_to(self, window, max_size): self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) -class MapRegion(object): +class MapRegion: """Defines a mapped region of memory, aligned to pagesizes @@ -198,7 +198,7 @@ class MapRegionList(list): ) def __new__(cls, path): - return super(MapRegionList, cls).__new__(cls) + return super().__new__(cls) def __init__(self, path_or_fd): self._path_or_fd = path_or_fd From db8810096503dd8a1f5a021ff39be907417f90a7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 15 Oct 2021 21:18:55 +0800 Subject: [PATCH 0469/1392] bump major version and update changelog --- doc/source/changes.rst | 7 +++++++ smmap/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index ff5ca9918..d0623b533 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,13 @@ Changelog ######### +****** +v5.0.0 +****** + +- Dropped support 3.5 +- Added support for Python 3.10 + ****** v4.0.0 ****** diff --git a/smmap/__init__.py b/smmap/__init__.py index 45f8abec8..696401c8c 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (4, 0, 0) +version_info = (5, 0, 0) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 47a74ca590efbdc3ae59277a491562efa7acc605 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 23 Oct 2021 08:29:50 +0800 Subject: [PATCH 0470/1392] Allow usage of smmap 5.0 (#76) --- gitdb/ext/smmap | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 30e93fee5..db8810096 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 30e93fee57286afae25c28a97ba65a9770f9a729 +Subproject commit db8810096503dd8a1f5a021ff39be907417f90a7 diff --git a/requirements.txt b/requirements.txt index 72957a243..1b2e11db7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -smmap>=3.0.1,<5 +smmap>=3.0.1,<6 From 9a289258074fbf92a64186274067a46f7b27666e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 23 Oct 2021 08:36:34 +0800 Subject: [PATCH 0471/1392] Drop support for python 3.4/3.5; make use of smmap 5.0 which does the same --- doc/source/changes.rst | 8 ++++++++ gitdb/__init__.py | 2 +- setup.py | 8 +++----- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 1a41d537b..de448dd4e 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,14 @@ Changelog ######### +***** +4.0.8 +***** + +* drop support for python 3.4 and 3.5 due to EOL +* Updated upper bound for smmap requirement in setup.py + (`#69 `_) + ***** 4.0.7 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index bef9696c7..0d936dba5 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -30,7 +30,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 7) +version_info = (4, 0, 8) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 522a88f16..251ea93ba 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 7) +version_info = (4, 0, 8) __version__ = '.'.join(str(i) for i in version_info) setup( @@ -20,9 +20,9 @@ packages=('gitdb', 'gitdb.db', 'gitdb.utils', 'gitdb.test'), license="BSD License", zip_safe=False, - install_requires=['smmap>=3.0.1,<5'], + install_requires=['smmap>=3.0.1,<6'], long_description="""GitDB is a pure-Python git object database""", - python_requires='>=3.4', + python_requires='>=3.6', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 5 - Production/Stable", @@ -35,8 +35,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", From 2913a6454c9dfc803679dc5f75315e2d821ee977 Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Wed, 5 Jan 2022 21:58:22 +0800 Subject: [PATCH 0472/1392] Fix typos --- gitdb/__init__.py | 2 +- gitdb/db/pack.py | 2 +- gitdb/fun.py | 8 ++++---- gitdb/pack.py | 4 ++-- gitdb/stream.py | 2 +- gitdb/test/db/lib.py | 2 +- gitdb/test/db/test_pack.py | 2 +- gitdb/test/test_pack.py | 2 +- gitdb/util.py | 6 +++--- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 0d936dba5..b5d1939dc 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -21,7 +21,7 @@ def _init_externals(): except ImportError as e: raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) from e # END verify import - # END handel imports + # END handle imports #} END initialization diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 177ed7bb2..90de02b61 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -131,7 +131,7 @@ def store(self, istream): def update_cache(self, force=False): """ - Update our cache with the acutally existing packs on disk. Add new ones, + Update our cache with the actually existing packs on disk. Add new ones, and remove deleted ones. We keep the unchanged ones :param force: If True, the cache will be updated even though the directory diff --git a/gitdb/fun.py b/gitdb/fun.py index 98465975d..abb4277db 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -103,7 +103,7 @@ def delta_chunk_apply(dc, bbuf, write): write(bbuf[dc.so:dc.so + dc.ts]) else: # APPEND DATA - # whats faster: if + 4 function calls or just a write with a slice ? + # what's faster: if + 4 function calls or just a write with a slice ? # Considering data can be larger than 127 bytes now, it should be worth it if dc.ts < len(dc.data): write(dc.data[:dc.ts]) @@ -292,7 +292,7 @@ def check_integrity(self, target_size=-1): """Verify the list has non-overlapping chunks only, and the total size matches target_size :param target_size: if not -1, the total size of the chain must be target_size - :raise AssertionError: if the size doen't match""" + :raise AssertionError: if the size doesn't match""" if target_size > -1: assert self[-1].rbound() == target_size assert reduce(lambda x, y: x + y, (d.ts for d in self), 0) == target_size @@ -331,7 +331,7 @@ def connect_with_next_base(self, bdcl): cannot be changed by any of the upcoming bases anymore. Once all our chunks are marked like that, we can stop all processing :param bdcl: data chunk list being one of our bases. They must be fed in - consequtively and in order, towards the earliest ancestor delta + consecutively and in order, towards the earliest ancestor delta :return: True if processing was done. Use it to abort processing of remaining streams if False is returned""" nfc = 0 # number of frozen chunks @@ -624,7 +624,7 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write): :param src_buf: random access data from which the delta was created :param src_buf_size: size of the source buffer in bytes - :param delta_buf_size: size fo the delta buffer in bytes + :param delta_buf_size: size for the delta buffer in bytes :param delta_buf: random access delta data :param write: write method taking a chunk of bytes diff --git a/gitdb/pack.py b/gitdb/pack.py index a38468e37..0b26c121c 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -224,7 +224,7 @@ def write(self, pack_sha, write): if ofs > 0x7fffffff: tmplist.append(ofs) ofs = 0x80000000 + len(tmplist) - 1 - # END hande 64 bit offsets + # END handle 64 bit offsets sha_write(pack('>L', ofs & 0xffffffff)) # END for each offset @@ -506,7 +506,7 @@ class PackFile(LazyMixin): """A pack is a file written according to the Version 2 for git packs As we currently use memory maps, it could be assumed that the maximum size of - packs therefor is 32 bit on 32 bit systems. On 64 bit systems, this should be + packs therefore is 32 bit on 32 bit systems. On 64 bit systems, this should be fine though. **Note:** at some point, this might be implemented using streams as well, or diff --git a/gitdb/stream.py b/gitdb/stream.py index d58d1a63e..37380ad3e 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -289,7 +289,7 @@ def read(self, size=-1): # They are thorough, and I assume it is truly working. # Why is this logic as convoluted as it is ? Please look at the table in # https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. - # Bascially, on py2.6, you want to use branch 1, whereas on all other python version, the second branch + # Basically, on py2.6, you want to use branch 1, whereas on all other python version, the second branch # will be the one that works. # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the # table in the github issue. This is it ... it was the only way I could make this work everywhere. diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index 3df326b68..b38f1d5e5 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -107,7 +107,7 @@ def _assert_object_writing(self, db): # DIRECT STREAM COPY # our data hase been written in object format to the StringIO - # we pasesd as output stream. No physical database representation + # we passed as output stream. No physical database representation # was created. # Test direct stream copy of object streams, the result must be # identical to what we fed in diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 458d80434..ff96a58ac 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -80,7 +80,7 @@ def test_writing(self, path): # END for each sha to find # we should have at least one ambiguous, considering the small sizes - # but in our pack, there is no ambigious ... + # but in our pack, there is no ambiguous ... # assert num_ambiguous # non-existing diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 48a185290..4b01741b2 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -101,7 +101,7 @@ def _assert_pack_file(self, pack, version, size): dstream = DeltaApplyReader.new(streams) except ValueError: # ignore these, old git versions use only ref deltas, - # which we havent resolved ( as we are without an index ) + # which we haven't resolved ( as we are without an index ) # Also ignore non-delta streams continue # END get deltastream diff --git a/gitdb/util.py b/gitdb/util.py index c4cafecc6..f9f8c0ed7 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -182,7 +182,7 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): pass # END exception handling - # read manully + # read manually contents = os.read(fd, os.fstat(fd).st_size) if stream: return _RandomAccessBytesIO(contents) @@ -248,7 +248,7 @@ class LazyMixin(object): def __getattr__(self, attr): """ Whenever an attribute is requested that we do not know, we allow it - to be created and set. Next time the same attribute is reqeusted, it is simply + to be created and set. Next time the same attribute is requested, it is simply returned from our dict/slots. """ self._set_cache_(attr) # will raise in case the cache was not created @@ -332,7 +332,7 @@ def open(self, write=False, stream=False): # open actual file if required if self._fd is None: - # we could specify exlusive here, as we obtained the lock anyway + # we could specify exclusive here, as we obtained the lock anyway try: self._fd = os.open(self._filepath, os.O_RDONLY | binary) except: From 597d36c4ac84af6deb7cecf2a1e8b4ca4b7dccdf Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Sat, 15 Jan 2022 23:28:12 +0800 Subject: [PATCH 0473/1392] Fix typos --- smmap/__init__.py | 2 +- smmap/buf.py | 2 +- smmap/mman.py | 10 +++++----- smmap/test/test_buf.py | 2 +- smmap/test/test_mman.py | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/smmap/__init__.py b/smmap/__init__.py index 696401c8c..6b1a44cb5 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -1,4 +1,4 @@ -"""Intialize the smmap package""" +"""Initialize the smmap package""" __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" diff --git a/smmap/buf.py b/smmap/buf.py index 795e0fd87..ad27b6974 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -21,7 +21,7 @@ class SlidingWindowMapBuffer: ) def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0): - """Initalize the instance to operate on the given cursor. + """Initialize the instance to operate on the given cursor. :param cursor: if not None, the associated cursor to the file you want to access If None, you have call begin_access before using the buffer and provide a cursor :param offset: absolute offset in bytes diff --git a/smmap/mman.py b/smmap/mman.py index 1de7d9e97..873f687fd 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -27,7 +27,7 @@ class WindowCursor: that it must be suited for the somewhat quite different sliding manager. It could be improved, but I see no real need to do so.""" __slots__ = ( - '_manager', # the manger keeping all file regions + '_manager', # the manager keeping all file regions '_rlist', # a regions list with regions for our file '_region', # our current class:`MapRegion` or None '_ofs', # relative offset from the actually mapped area to our start area @@ -66,7 +66,7 @@ def _destroy(self): # sometimes, during shutdown, getrefcount is None. Its possible # to re-import it, however, its probably better to just ignore # this python problem (for now). - # The next step is to get rid of the error prone getrefcount alltogether. + # The next step is to get rid of the error prone getrefcount altogether. pass # END exception handling # END handle regions @@ -95,7 +95,7 @@ def __copy__(self): #{ Interface def assign(self, rhs): """Assign rhs to this instance. This is required in order to get a real copy. - Alternativly, you can copy an existing instance using the copy module""" + Alternatively, you can copy an existing instance using the copy module""" self._destroy() self._copy_from(rhs) @@ -342,7 +342,7 @@ def _collect_lru_region(self, size): return num_found def _obtain_region(self, a, offset, size, flags, is_recursive): - """Utilty to create a new region - for more information on the parameters, + """Utility to create a new region - for more information on the parameters, see MapCursor.use_region. :param a: A regions (a)rray :return: The newly created region""" @@ -427,7 +427,7 @@ def mapped_memory_size(self): return self._memory_size def max_file_handles(self): - """:return: maximium amount of handles we may have opened""" + """:return: maximum amount of handles we may have opened""" return self._max_handle_count def max_mapped_memory_size(self): diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index 17555afe4..f0a86fb64 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -30,7 +30,7 @@ def test_basics(self): self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large - buf = SlidingWindowMapBuffer() # can create uninitailized buffers + buf = SlidingWindowMapBuffer() # can create uninitialized buffers assert buf.cursor() is None # can call end access any time diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index d88316b8e..7a5f4092c 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -154,7 +154,7 @@ def test_memman_operation(self): if man.window_size(): assert man.num_file_handles() == 2 assert c.size() < size - assert c.region() is not rr # old region is still available, but has not curser ref anymore + assert c.region() is not rr # old region is still available, but has not cursor ref anymore assert rr.client_count() == 1 # only held by manager else: assert c.size() < fc.size @@ -194,7 +194,7 @@ def test_memman_operation(self): # precondition if man.window_size(): assert max_mapped_memory_size >= mapped_memory_size() - # END statics will overshoot, which is fine + # END statistics will overshoot, which is fine assert max_file_handles >= num_file_handles() assert c.use_region(base_offset, (size or c.size())).is_valid() csize = c.size() @@ -205,7 +205,7 @@ def test_memman_operation(self): assert includes_ofs(base_offset + csize - 1) assert not includes_ofs(base_offset + csize) # END while we should do an access - elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows + elapsed = max(time() - st, 0.001) # prevent zero division errors on windows mb = float(1000 * 1000) print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" % (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed), From e3a4cfe0ef2bc7b9c785c4fec650f5045cdd1e50 Mon Sep 17 00:00:00 2001 From: Carl George Date: Wed, 9 Feb 2022 17:15:39 -0600 Subject: [PATCH 0474/1392] Switch from nose to pytest This is not a full rewrite to pytest style tests, it just changes the minimum to allow pytest to run the existing tests. Resolves #72 --- .github/workflows/pythonpackage.yml | 6 +++--- Makefile | 3 +-- README.rst | 4 ++-- gitdb.pro.user | 3 +-- gitdb/test/db/test_pack.py | 4 ++-- gitdb/test/lib.py | 4 ++-- gitdb/test/test_pack.py | 4 ++-- 7 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 7f35b616e..c695ad302 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -36,9 +36,9 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with nose + - name: Test with pytest run: | - pip install nose + pip install pytest ulimit -n 48 ulimit -n - nosetests -v + pytest -v diff --git a/Makefile b/Makefile index 82907561b..e0630283d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ PYTHON = python SETUP = $(PYTHON) setup.py -TESTRUNNER = $(shell which nosetests) TESTFLAGS = all:: @@ -37,5 +36,5 @@ clean:: rm -f *.so coverage:: build - PYTHONPATH=. $(PYTHON) $(TESTRUNNER) --cover-package=gitdb --with-coverage --cover-erase --cover-inclusive gitdb + PYTHONPATH=. $(PYTHON) -m pytest --cov=gitdb gitdb diff --git a/README.rst b/README.rst index 44b3eddf8..29c70f781 100644 --- a/README.rst +++ b/README.rst @@ -30,7 +30,7 @@ If you want to go up to 20% faster, you can install gitdb-speedups with: REQUIREMENTS ============ -* Python Nose - for running the tests +* pytest - for running the tests SOURCE ====== @@ -45,7 +45,7 @@ Once the clone is complete, please be sure to initialize the submodules using Run the tests with - nosetests + pytest DEVELOPMENT =========== diff --git a/gitdb.pro.user b/gitdb.pro.user index 398cb70a1..3ca1e2182 100644 --- a/gitdb.pro.user +++ b/gitdb.pro.user @@ -233,8 +233,7 @@ - /usr/bin/nosetests - -s + /usr/bin/pytest gitdb/test/test_pack.py 2 diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index ff96a58ac..4539f4278 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -16,7 +16,7 @@ import random import sys -from nose.plugins.skip import SkipTest +import pytest class TestPackDB(TestDBBase): @@ -24,7 +24,7 @@ class TestPackDB(TestDBBase): @with_packs_rw def test_writing(self, path): if sys.platform == "win32": - raise SkipTest("FIXME: Currently fail on windows") + pytest.skip("FIXME: Currently fail on windows") pdb = PackedDB(path) diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index a04084f5f..abd4ad57d 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -65,8 +65,8 @@ def skip_on_travis_ci(func): @wraps(func) def wrapper(self, *args, **kwargs): if 'TRAVIS' in os.environ: - import nose - raise nose.SkipTest("Cannot run on travis-ci") + import pytest + pytest.skip("Cannot run on travis-ci") # end check for travis ci return func(self, *args, **kwargs) # end wrapper diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index 4b01741b2..f9461975b 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -26,7 +26,7 @@ from gitdb.exc import UnsupportedOperation from gitdb.util import to_bin_sha -from nose import SkipTest +import pytest import os import tempfile @@ -246,4 +246,4 @@ def rewind_streams(): def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets # of course without really needing such a huge pack - raise SkipTest() + pytest.skip('not implemented') From 7c67010acf541b4269825cb2c13e226fe2d65ea9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 24 Oct 2021 21:38:51 +0800 Subject: [PATCH 0475/1392] bump patch level in the hopes for a valid sig on release https://github.com/gitpython-developers/gitdb/issues/77 --- doc/source/changes.rst | 6 ++++++ gitdb/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index de448dd4e..5ef1326d0 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +***** +4.0.9 +***** + +- re-release of 4.0.8 to get a valid signature. + ***** 4.0.8 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index b5d1939dc..246014554 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -30,7 +30,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 8) +version_info = (4, 0, 9) __version__ = '.'.join(str(i) for i in version_info) diff --git a/setup.py b/setup.py index 251ea93ba..31dc62f47 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 8) +version_info = (4, 0, 9) __version__ = '.'.join(str(i) for i in version_info) setup( From 4762d99d978586fcdf08ade552f4712bfde6ef22 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 21 Feb 2022 10:43:54 +0800 Subject: [PATCH 0476/1392] Stop signing releases (#77) The key stopped producing correct signatures when upgrading MacOS, at least when used by `twine`. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e0630283d..a7acbb10e 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ release:: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i 27C50E7F590947D7273A741E85194C08421980C9 dist/* + twine upload dist/* doc:: make -C doc/ html From 334ef84a05c953ed5dbec7b9c6d4310879eeab5a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 21 Feb 2022 10:45:33 +0800 Subject: [PATCH 0477/1392] Stop signing releases Related to https://github.com/gitpython-developers/gitdb/issues/77 --- Makefile | 2 +- smmap/util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9145cabb0..051e99ba0 100644 --- a/Makefile +++ b/Makefile @@ -46,4 +46,4 @@ release: clean force_release:: clean git push --tags python3 setup.py sdist bdist_wheel - twine upload -s -i 27C50E7F590947D7273A741E85194C08421980C9 dist/* + twine upload dist/* diff --git a/smmap/util.py b/smmap/util.py index cf027afdc..fbb387211 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -71,7 +71,7 @@ def extend_left_to(self, window, max_size): rofs = self.ofs - window.ofs_end() nsize = rofs + self.size rofs -= nsize - min(nsize, max_size) - self.ofs = self.ofs - rofs + self.ofs -= rofs self.size += rofs def extend_right_to(self, window, max_size): From 2ce0e3175bcbbf397d25f18b1008d1fdf54611f2 Mon Sep 17 00:00:00 2001 From: randymcmillan Date: Tue, 15 Mar 2022 17:51:07 -0400 Subject: [PATCH 0478/1392] .gitmodules: update smmap url Github.com is currenly redirecting the smmap url: https://github.com/Byron/smmap.git to: https://github.com/gitpython-developers/smmap For correctness we update to the url directly --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index d85b15c9f..e73cdedab 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "smmap"] path = gitdb/ext/smmap - url = https://github.com/Byron/smmap.git + url = https://github.com/gitpython-developers/smmap.git From a8cdf46b26f2f7029d3749bc431e2b80fe28b04a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:04:02 +0200 Subject: [PATCH 0479/1392] Add support for Python 3.11 --- .github/workflows/pythonpackage.yml | 6 +++--- tox.ini | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index e6e68ff7c..d593fd066 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,14 +15,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/tox.ini b/tox.ini index c34ab0290..092c79daa 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py36, py37, py38, py39, py310 +envlist = flake8, py{36, 37, 38, 39, 310, 311} [testenv] commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} From 0370014ee74a0f4480127932a78f177f6f433b9e Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:06:32 +0200 Subject: [PATCH 0480/1392] Drop support for EOL Python 3.6 --- .github/workflows/pythonpackage.yml | 2 +- README.md | 2 +- doc/source/intro.rst | 2 +- setup.py | 3 +-- smmap/buf.py | 2 +- tox.ini | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index d593fd066..9e2d881de 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/README.md b/README.md index f083dd004..e21f53453 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 3.6+ +* Python 3.7+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 3489b04ee..109fec247 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -20,7 +20,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 3.6+ +* Python 3.7+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/setup.py b/setup.py index 9ab8a7f0c..1df3b62aa 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, - python_requires=">=3.6", + python_requires=">=3.7", classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -38,7 +38,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", diff --git a/smmap/buf.py b/smmap/buf.py index ad27b6974..731b0644b 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -93,7 +93,7 @@ def __getslice__(self, i, j): d = d.tobytes() md.append(d) # END while there are bytes to read - return bytes().join(md) + return b''.join(md) # END fast or slow path #{ Interface diff --git a/tox.ini b/tox.ini index 092c79daa..810baddc3 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py{36, 37, 38, 39, 310, 311} +envlist = flake8, py{37, 38, 39, 310, 311} [testenv] commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} From 5f149c1a9c36c985158c0757377af958362b3582 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:13:01 +0200 Subject: [PATCH 0481/1392] Add support for Python 3.10 and 3.11 --- .github/workflows/pythonpackage.yml | 6 +++--- setup.py | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index c695ad302..3c05764f5 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,14 +15,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + python-version: ["3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/setup.py b/setup.py index 31dc62f47..5d324cd96 100755 --- a/setup.py +++ b/setup.py @@ -39,5 +39,8 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", ] ) From a6f0856d807efc1e7bc37d13f9cfbcdb91dea2ac Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:15:02 +0200 Subject: [PATCH 0482/1392] Remove redundant Travis CI config/code --- .travis.yml | 20 ------------------- gitdb/test/lib.py | 15 -------------- gitdb/test/performance/test_pack.py | 4 ---- gitdb/test/performance/test_pack_streaming.py | 3 --- gitdb/test/performance/test_stream.py | 2 -- 5 files changed, 44 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b980d36e9..000000000 --- a/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ -# NOT USED, just for reference. See github actions for CI configuration -language: python -python: - - "3.4" - - "3.5" - - "3.6" - # - "pypy" - won't work as smmap doesn't work (see smmap/.travis.yml for details) - -git: - # a higher depth is needed for one of the tests - lets fet - depth: 1000 -install: - - pip install coveralls -script: - - ulimit -n 48 - - ulimit -n - - nosetests -v --with-coverage -after_success: - - coveralls - diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index abd4ad57d..465a899a9 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -58,21 +58,6 @@ def setUpClass(cls): #{ Decorators -def skip_on_travis_ci(func): - """All tests decorated with this one will raise SkipTest when run on travis ci. - Use it to workaround difficult to solve issues - NOTE: copied from bcore (https://github.com/Byron/bcore)""" - @wraps(func) - def wrapper(self, *args, **kwargs): - if 'TRAVIS' in os.environ: - import pytest - pytest.skip("Cannot run on travis-ci") - # end check for travis ci - return func(self, *args, **kwargs) - # end wrapper - return wrapper - - def with_rw_directory(func): """Create a temporary directory which can be written to, remove it if the test succeeds, but leave it otherwise to aid additional debugging""" diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index b59d5a971..643186b06 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -17,7 +17,6 @@ from gitdb.typ import str_blob_type from gitdb.exc import UnsupportedOperation from gitdb.db.pack import PackedDB -from gitdb.test.lib import skip_on_travis_ci import sys import os @@ -26,7 +25,6 @@ class TestPackedDBPerformance(TestBigRepoR): - @skip_on_travis_ci def test_pack_random_access(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) @@ -79,7 +77,6 @@ def test_pack_random_access(self): print("PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib / (elapsed or 1), elapsed, max_items / (elapsed or 1)), file=sys.stderr) - @skip_on_travis_ci def test_loose_correctness(self): """based on the pack(s) of our packed object DB, we will just copy and verify all objects in the back into the loose object db (memory). @@ -106,7 +103,6 @@ def test_loose_correctness(self): mdb._cache.clear() # end for each sha to copy - @skip_on_travis_ci def test_correctness(self): pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) # disabled for now as it used to work perfectly, checking big repositories takes a long time diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 76f0f4a07..5bf67909b 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -12,7 +12,6 @@ from gitdb.db.pack import PackedDB from gitdb.stream import NullStream from gitdb.pack import PackEntity -from gitdb.test.lib import skip_on_travis_ci import os import sys @@ -34,7 +33,6 @@ def write(self, d): class TestPackStreamingPerformance(TestBigRepoR): - @skip_on_travis_ci def test_pack_writing(self): # see how fast we can write a pack from object streams. # This will not be fast, as we take time for decompressing the streams as well @@ -61,7 +59,6 @@ def test_pack_writing(self): print(sys.stderr, "PDB Streaming: Wrote pack of size %i kb in %f s (%f kb/s)" % (total_kb, elapsed, total_kb / (elapsed or 1)), sys.stderr) - @skip_on_travis_ci def test_stream_reading(self): # raise SkipTest() pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack")) diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 92d28e4a3..9a8b15b2d 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -20,7 +20,6 @@ from gitdb.test.lib import ( make_memory_file, with_rw_directory, - skip_on_travis_ci ) @@ -44,7 +43,6 @@ class TestObjDBPerformance(TestBigRepoR): large_data_size_bytes = 1000 * 1000 * 50 # some MiB should do it moderate_data_size_bytes = 1000 * 1000 * 1 # just 1 MiB - @skip_on_travis_ci @with_rw_directory def test_large_data_streaming(self, path): ldb = LooseObjectDB(path) From faed217d14965932b333c07219ed401a661d2651 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:16:09 +0200 Subject: [PATCH 0483/1392] Drop support for EOL Python 3.5 and 3.6 --- .github/workflows/pythonpackage.yml | 2 +- setup.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3c05764f5..0d039adae 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/setup.py b/setup.py index 5d324cd96..d38b267ce 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ zip_safe=False, install_requires=['smmap>=3.0.1,<6'], long_description="""GitDB is a pure-Python git object database""", - python_requires='>=3.6', + python_requires='>=3.7', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 5 - Production/Stable", @@ -35,7 +35,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", From 7a68270d6c78b81f577b433dc6351b26bc27b7cf Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:17:20 +0200 Subject: [PATCH 0484/1392] Upgrade Python syntax with pyupgrade --py37-plus --- doc/source/conf.py | 9 ++++----- gitdb/db/base.py | 12 ++++++------ gitdb/db/git.py | 4 ++-- gitdb/db/loose.py | 4 ++-- gitdb/db/mem.py | 2 +- gitdb/db/pack.py | 2 +- gitdb/db/ref.py | 8 ++++---- gitdb/fun.py | 2 +- gitdb/pack.py | 4 ++-- gitdb/stream.py | 12 ++++++------ gitdb/test/db/test_ref.py | 2 +- gitdb/test/lib.py | 6 +++--- gitdb/test/performance/test_pack.py | 1 - gitdb/test/performance/test_pack_streaming.py | 1 - gitdb/test/performance/test_stream.py | 1 - gitdb/test/test_example.py | 2 +- gitdb/test/test_stream.py | 4 ++-- gitdb/util.py | 14 +++++++------- 18 files changed, 43 insertions(+), 47 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 3ab15ab35..b387f6094 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # GitDB documentation build configuration file, created by # sphinx-quickstart on Wed Jun 30 00:01:32 2010. @@ -38,8 +37,8 @@ master_doc = 'index' # General information about the project. -project = u'GitDB' -copyright = u'2011, Sebastian Thiel' +project = 'GitDB' +copyright = '2011, Sebastian Thiel' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -172,8 +171,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'GitDB.tex', u'GitDB Documentation', - u'Sebastian Thiel', 'manual'), + ('index', 'GitDB.tex', 'GitDB Documentation', + 'Sebastian Thiel', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/gitdb/db/base.py b/gitdb/db/base.py index f0b8a0574..e89052e88 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -22,7 +22,7 @@ __all__ = ('ObjectDBR', 'ObjectDBW', 'FileDBBase', 'CompoundDB', 'CachingDB') -class ObjectDBR(object): +class ObjectDBR: """Defines an interface for object database lookup. Objects are identified either by their 20 byte bin sha""" @@ -63,7 +63,7 @@ def sha_iter(self): #} END query interface -class ObjectDBW(object): +class ObjectDBW: """Defines an interface to create objects in the database""" @@ -105,7 +105,7 @@ def store(self, istream): #} END edit interface -class FileDBBase(object): +class FileDBBase: """Provides basic facilities to retrieve files of interest, including caching facilities to help mapping hexsha's to objects""" @@ -117,7 +117,7 @@ def __init__(self, root_path): **Note:** The base will not perform any accessablity checking as the base might not yet be accessible, but become accessible before the first access.""" - super(FileDBBase, self).__init__() + super().__init__() self._root_path = root_path #{ Interface @@ -133,7 +133,7 @@ def db_path(self, rela_path): #} END interface -class CachingDB(object): +class CachingDB: """A database which uses caches to speed-up access""" @@ -176,7 +176,7 @@ def _set_cache_(self, attr): elif attr == '_db_cache': self._db_cache = dict() else: - super(CompoundDB, self)._set_cache_(attr) + super()._set_cache_(attr) def _db_query(self, sha): """:return: database containing the given 20 byte sha diff --git a/gitdb/db/git.py b/gitdb/db/git.py index 7a43d7235..e2cb46805 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -39,7 +39,7 @@ class GitDB(FileDBBase, ObjectDBW, CompoundDB): def __init__(self, root_path): """Initialize ourselves on a git objects directory""" - super(GitDB, self).__init__(root_path) + super().__init__(root_path) def _set_cache_(self, attr): if attr == '_dbs' or attr == '_loose_db': @@ -68,7 +68,7 @@ def _set_cache_(self, attr): # finally set the value self._loose_db = loose_db else: - super(GitDB, self)._set_cache_(attr) + super()._set_cache_(attr) # END handle attrs #{ ObjectDBW interface diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index a63a2ef33..4ef775049 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -75,7 +75,7 @@ class LooseObjectDB(FileDBBase, ObjectDBR, ObjectDBW): new_objects_mode = int("644", 8) def __init__(self, root_path): - super(LooseObjectDB, self).__init__(root_path) + super().__init__(root_path) self._hexsha_to_file = dict() # Additional Flags - might be set to 0 after the first failure # Depending on the root, this might work for some mounts, for others not, which @@ -151,7 +151,7 @@ def set_ostream(self, stream): """:raise TypeError: if the stream does not support the Sha1Writer interface""" if stream is not None and not isinstance(stream, Sha1Writer): raise TypeError("Output stream musst support the %s interface" % Sha1Writer.__name__) - return super(LooseObjectDB, self).set_ostream(stream) + return super().set_ostream(stream) def info(self, sha): m = self._map_loose_object(sha) diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index b2542ff1b..1b954cba2 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -37,7 +37,7 @@ class MemoryDB(ObjectDBR, ObjectDBW): exists in the target storage before introducing actual IO""" def __init__(self): - super(MemoryDB, self).__init__() + super().__init__() self._db = LooseObjectDB("path/doesnt/matter") # maps 20 byte shas to their OStream objects diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 90de02b61..1ce786b79 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -39,7 +39,7 @@ class PackedDB(FileDBBase, ObjectDBR, CachingDB, LazyMixin): _sort_interval = 500 def __init__(self, root_path): - super(PackedDB, self).__init__(root_path) + super().__init__(root_path) # list of lists with three items: # * hits - number of times the pack was hit with a request # * entity - Pack entity instance diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 2bb1de790..6bb2a64dd 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -20,7 +20,7 @@ class ReferenceDB(CompoundDB): ObjectDBCls = None def __init__(self, ref_file): - super(ReferenceDB, self).__init__() + super().__init__() self._ref_file = ref_file def _set_cache_(self, attr): @@ -28,7 +28,7 @@ def _set_cache_(self, attr): self._dbs = list() self._update_dbs_from_ref_file() else: - super(ReferenceDB, self)._set_cache_(attr) + super()._set_cache_(attr) # END handle attrs def _update_dbs_from_ref_file(self): @@ -44,7 +44,7 @@ def _update_dbs_from_ref_file(self): try: with codecs.open(self._ref_file, 'r', encoding="utf-8") as f: ref_paths = [l.strip() for l in f] - except (OSError, IOError): + except OSError: pass # END handle alternates @@ -79,4 +79,4 @@ def _update_dbs_from_ref_file(self): def update_cache(self, force=False): # re-read alternates and update databases self._update_dbs_from_ref_file() - return super(ReferenceDB, self).update_cache(force) + return super().update_cache(force) diff --git a/gitdb/fun.py b/gitdb/fun.py index abb4277db..a4454de4d 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -113,7 +113,7 @@ def delta_chunk_apply(dc, bbuf, write): # END handle chunk mode -class DeltaChunk(object): +class DeltaChunk: """Represents a piece of a delta, it can either add new data, or copy existing one from a source buffer""" diff --git a/gitdb/pack.py b/gitdb/pack.py index 0b26c121c..cabce4c9d 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -170,7 +170,7 @@ def write_stream_to_pack(read, write, zstream, base_crc=None): #} END utilities -class IndexWriter(object): +class IndexWriter: """Utility to cache index information, allowing to write all information later in one go to the given stream @@ -257,7 +257,7 @@ class PackIndexFile(LazyMixin): index_version_default = 2 def __init__(self, indexpath): - super(PackIndexFile, self).__init__() + super().__init__() self._indexpath = indexpath def close(self): diff --git a/gitdb/stream.py b/gitdb/stream.py index 37380ad3e..222b843e9 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -219,13 +219,13 @@ def read(self, size=-1): # END clamp size if size == 0: - return bytes() + return b'' # END handle depletion # deplete the buffer, then just continue using the decompress object # which has an own buffer. We just need this to transparently parse the # header from the zlib stream - dat = bytes() + dat = b'' if self._buf: if self._buflen >= size: # have enough data @@ -553,7 +553,7 @@ def size(self): #{ W Streams -class Sha1Writer(object): +class Sha1Writer: """Simple stream writer which produces a sha whenever you like as it degests everything it is supposed to write""" @@ -650,7 +650,7 @@ class FDCompressedSha1Writer(Sha1Writer): exc = IOError("Failed to write all bytes to filedescriptor") def __init__(self, fd): - super(FDCompressedSha1Writer, self).__init__() + super().__init__() self.fd = fd self.zip = zlib.compressobj(zlib.Z_BEST_SPEED) @@ -677,7 +677,7 @@ def close(self): #} END stream interface -class FDStream(object): +class FDStream: """A simple wrapper providing the most basic functions on a file descriptor with the fileobject interface. Cannot use os.fdopen as the resulting stream @@ -711,7 +711,7 @@ def close(self): close(self._fd) -class NullStream(object): +class NullStream: """A stream that does nothing but providing a stream interface. Use it like /dev/null""" diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 20496985e..664aa54ba 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -23,7 +23,7 @@ def make_alt_file(self, alt_path, alt_list): The list can be empty""" with open(alt_path, "wb") as alt_file: for alt in alt_list: - alt_file.write(alt.encode("utf-8") + "\n".encode("ascii")) + alt_file.write(alt.encode("utf-8") + b"\n") @with_rw_directory def test_writing(self, path): diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index 465a899a9..da59d3b35 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -40,7 +40,7 @@ class TestBase(unittest.TestCase): @classmethod def setUpClass(cls): try: - super(TestBase, cls).setUpClass() + super().setUpClass() except AttributeError: pass @@ -70,7 +70,7 @@ def wrapper(self): try: return func(self, path) except Exception: - sys.stderr.write("Test {}.{} failed, output is at {!r}\n".format(type(self).__name__, func.__name__, path)) + sys.stderr.write(f"Test {type(self).__name__}.{func.__name__} failed, output is at {path!r}\n") keep = True raise finally: @@ -161,7 +161,7 @@ def make_memory_file(size_in_bytes, randomize=False): #{ Stream Utilities -class DummyStream(object): +class DummyStream: def __init__(self): self.was_read = False diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index 643186b06..f034baffd 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -3,7 +3,6 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance tests for object store""" -from __future__ import print_function from gitdb.test.performance.lib import ( TestBigRepoR diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index 5bf67909b..db790f1db 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -3,7 +3,6 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Specific test for pack streams only""" -from __future__ import print_function from gitdb.test.performance.lib import ( TestBigRepoR diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 9a8b15b2d..91dc89178 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -3,7 +3,6 @@ # This module is part of GitDB and is released under # the New BSD License: http://www.opensource.org/licenses/bsd-license.php """Performance data streaming performance""" -from __future__ import print_function from gitdb.test.performance.lib import TestBigRepoR from gitdb.db import LooseObjectDB diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index 6e80bf5c6..cc4d40dd3 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -32,7 +32,7 @@ def test_base(self): pass # END ignore exception if there are no loose objects - data = "my data".encode("ascii") + data = b"my data" istream = IStream("blob", len(data), BytesIO(data)) # the object does not yet have a sha diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 5d4b93db7..5e2e1bab2 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -115,13 +115,13 @@ def test_decompress_reader(self): def test_sha_writer(self): writer = Sha1Writer() - assert 2 == writer.write("hi".encode("ascii")) + assert 2 == writer.write(b"hi") assert len(writer.sha(as_hex=1)) == 40 assert len(writer.sha(as_hex=0)) == 20 # make sure it does something ;) prev_sha = writer.sha() - writer.write("hi again".encode("ascii")) + writer.write(b"hi again") assert writer.sha() != prev_sha def test_compressed_writer(self): diff --git a/gitdb/util.py b/gitdb/util.py index f9f8c0ed7..3151c061e 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -94,7 +94,7 @@ def remove(*args, **kwargs): #{ compatibility stuff ... -class _RandomAccessBytesIO(object): +class _RandomAccessBytesIO: """Wrapper to provide required functionality in case memory maps cannot or may not be used. This is only really required in python 2.4""" @@ -131,7 +131,7 @@ def byte_ord(b): #{ Routines -def make_sha(source=''.encode("ascii")): +def make_sha(source=b''): """A python2.4 workaround for the sha/hashlib module fiasco **Note** From the dulwich project """ @@ -151,7 +151,7 @@ def allocate_memory(size): try: return mmap.mmap(-1, size) # read-write by default - except EnvironmentError: + except OSError: # setup real memory instead # this of course may fail if the amount of memory is not available in # one chunk - would only be the case in python 2.4, being more likely on @@ -174,7 +174,7 @@ def file_contents_ro(fd, stream=False, allow_mmap=True): # supports stream and random access try: return mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - except EnvironmentError: + except OSError: # python 2.4 issue, 0 wants to be the actual size return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ) # END handle python 2.4 @@ -234,7 +234,7 @@ def to_bin_sha(sha): #{ Utilities -class LazyMixin(object): +class LazyMixin: """ Base class providing an interface to lazily retrieve attribute values upon @@ -266,7 +266,7 @@ def _set_cache_(self, attr): pass -class LockedFD(object): +class LockedFD: """ This class facilitates a safe read and write operation to a file on disk. @@ -327,7 +327,7 @@ def open(self, write=False, stream=False): self._fd = fd # END handle file descriptor except OSError as e: - raise IOError("Lock at %r could not be obtained" % self._lockfilepath()) from e + raise OSError("Lock at %r could not be obtained" % self._lockfilepath()) from e # END handle lock retrieval # open actual file if required From c3ab5d7b28062848c2a639a60e0acfbaee7e8f90 Mon Sep 17 00:00:00 2001 From: zwimer Date: Tue, 22 Nov 2022 18:57:47 -0700 Subject: [PATCH 0485/1392] Prefer import to __import__ --- .gitignore | 1 + gitdb/__init__.py | 15 ++++++--------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index e0b4e8579..8b7da9277 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ dist/ *.so .noseids *.sublime-workspace +*.egg-info diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 246014554..c5b554746 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -12,15 +12,12 @@ def _init_externals(): """Initialize external projects by putting them into the path""" - for module in ('smmap',): - if 'PYOXIDIZER' not in os.environ: - sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module)) - - try: - __import__(module) - except ImportError as e: - raise ImportError("'%s' could not be imported, assure it is located in your PYTHONPATH" % module) from e - # END verify import + if 'PYOXIDIZER' not in os.environ: + where = os.path.join(os.path.dirname(__file__), 'ext', 'smmap') + if os.path.exists(where): + sys.path.append(where) + + import smmap # END handle imports #} END initialization From 1edc7d296af635dc31030a09e73fd684eedc1d59 Mon Sep 17 00:00:00 2001 From: zwimer Date: Tue, 22 Nov 2022 19:05:11 -0700 Subject: [PATCH 0486/1392] Add del smmap to match previous behavior --- gitdb/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index c5b554746..94b0831ac 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -18,6 +18,7 @@ def _init_externals(): sys.path.append(where) import smmap + del smmap # END handle imports #} END initialization From 38c68d95eaed9bf7415781ca0102441a349893ee Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Nov 2022 06:22:21 +0100 Subject: [PATCH 0487/1392] bump version to 4.0.10 --- doc/source/changes.rst | 6 ++++++ gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 5ef1326d0..7d85ea55b 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +****** +4.0.10 +****** + +- improvements to the way external packages are imported. + ***** 4.0.9 ***** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 94b0831ac..b777632bb 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -28,7 +28,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 9) +version_info = (4, 0, 10) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index db8810096..334ef84a0 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit db8810096503dd8a1f5a021ff39be907417f90a7 +Subproject commit 334ef84a05c953ed5dbec7b9c6d4310879eeab5a diff --git a/setup.py b/setup.py index d38b267ce..521484978 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 9) +version_info = (4, 0, 10) __version__ = '.'.join(str(i) for i in version_info) setup( From 49c3178711ddb3510f0e96297187f823cc019871 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Nov 2022 06:24:37 +0100 Subject: [PATCH 0488/1392] use python3 binary MacOS Ventura doesn't come with a `python` binary anymore --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a7acbb10e..7aa5a717a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PYTHON = python +PYTHON = python3 SETUP = $(PYTHON) setup.py TESTFLAGS = From b6faecc46fcc4f6412149f2021447c0070eba60e Mon Sep 17 00:00:00 2001 From: Ondrej Tethal Date: Fri, 9 Dec 2022 10:21:58 +0100 Subject: [PATCH 0489/1392] Use ZLIB_RUNTIME_VERSION if available --- gitdb/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index 222b843e9..1b5426f02 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -294,7 +294,7 @@ def read(self, size=-1): # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the # table in the github issue. This is it ... it was the only way I could make this work everywhere. # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . - if zlib.ZLIB_VERSION in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': + if getattr(zlib, 'ZLIB_RUNTIME_VERSION', zlib.ZLIB_VERSION) in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': unused_datalen = len(self._zip.unconsumed_tail) else: unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) From 05229570e8e20a1129464af2f7cb3698f3aa9b48 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 10 Sep 2023 00:12:05 +0300 Subject: [PATCH 0490/1392] Bump GitHub Actions --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 9e2d881de..33c6eabe5 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -18,7 +18,7 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} From 8aa59e702c198e4efd578ca74dda581f73ee258c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 10 Sep 2023 00:13:08 +0300 Subject: [PATCH 0491/1392] Add support for Python 3.12 --- .github/workflows/pythonpackage.yml | 3 ++- setup.py | 2 ++ tox.ini | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 33c6eabe5..73613588d 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -25,6 +25,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/setup.py b/setup.py index 1df3b62aa..49d90a02d 100755 --- a/setup.py +++ b/setup.py @@ -42,6 +42,8 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3 :: Only", ], long_description=long_description, diff --git a/tox.ini b/tox.ini index 810baddc3..f7205c5a3 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py{37, 38, 39, 310, 311} +envlist = flake8, py{37, 38, 39, 310, 311, 312} [testenv] commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} From f1b4d14e7d3902135af336ed06a12308dfd40270 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 10 Sep 2023 00:14:52 +0300 Subject: [PATCH 0492/1392] Drop support for EOL Python 3.7 --- .github/workflows/pythonpackage.yml | 2 +- README.md | 2 +- doc/source/intro.rst | 2 +- setup.py | 3 +-- tox.ini | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 73613588d..55c654269 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index e21f53453..ff973bae6 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 3.7+ +* Python 3.8+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 109fec247..8f36481c2 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -20,7 +20,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 3.7+ +* Python 3.8+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/setup.py b/setup.py index 49d90a02d..b99532814 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, - python_requires=">=3.7", + python_requires=">=3.8", classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -38,7 +38,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tox.ini b/tox.ini index f7205c5a3..ac3536caa 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py{37, 38, 39, 310, 311, 312} +envlist = flake8, py{38, 39, 310, 311, 312} [testenv] commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} From 32d12aa03aff193603aad1d6f08669a70607beb9 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 10 Sep 2023 18:38:38 +0300 Subject: [PATCH 0493/1392] Bump GitHub Actions --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 0d039adae..1cefabc3e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -18,7 +18,7 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} From d7fc1fd3d154c809b1c021ae3fa564e9fe4b4859 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 10 Sep 2023 18:38:56 +0300 Subject: [PATCH 0494/1392] Add support for Python 3.12 --- .github/workflows/pythonpackage.yml | 3 ++- setup.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 1cefabc3e..dab41d093 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -25,6 +25,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/setup.py b/setup.py index 521484978..61b572768 100755 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3 :: Only", ] ) From 875acb45cd8e9353c1911717bd2cd05b3e40ed05 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 10 Sep 2023 18:39:33 +0300 Subject: [PATCH 0495/1392] Drop support for EOL Python 3.7 --- .github/workflows/pythonpackage.yml | 2 +- setup.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index dab41d093..98e670c22 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/setup.py b/setup.py index 61b572768..57341223b 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ zip_safe=False, install_requires=['smmap>=3.0.1,<6'], long_description="""GitDB is a pure-Python git object database""", - python_requires='>=3.7', + python_requires='>=3.8', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 5 - Production/Stable", @@ -35,7 +35,6 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", From 8ac188497e7ac1ef2e89c984ac3728319b625e1a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Sep 2023 18:14:06 -0400 Subject: [PATCH 0496/1392] Enable Dependabot version updates for Actions This enables Dependabot version updates for GitHub Actions only (not Python dependencies), using the exact same configuration as in GitPython. --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..203f3c889 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" From e7937aefaf49c72b5f4432cc1f303ed23889589a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 02:31:34 -0400 Subject: [PATCH 0497/1392] Test installing project on CI This changes the CI workflow to install the project itself to get its dependency, rather than installing the dependency from requirements.txt. This is the more important thing to test, because it verifies that the project is installable and effectively declares the dependencies it needs. --- .github/workflows/pythonpackage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 98e670c22..efa04a901 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -26,10 +26,10 @@ jobs: with: python-version: ${{ matrix.python-version }} allow-prereleases: true - - name: Install dependencies + - name: Install project and dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install . - name: Lint with flake8 run: | pip install flake8 From 60f7f94607996e05292896e8105ef3780779959d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 03:38:59 -0400 Subject: [PATCH 0498/1392] Fix mkdir race condition in LooseObjectDB.store This replaces the conditional call to os.mkdir that raises an unintended FileExistsError if the directory is created between the check and the os.mkdir call, using a single os.makedirs call instead, with exist_ok=True. This way, we attempt creation in a way that produces no error if the directory is already present, while still raising FileExistsError if a non-directory filesystem entry (such as a regular file) is present where we want the directory to be. --- gitdb/db/loose.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 4ef775049..7ea6fef3d 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -8,7 +8,6 @@ ObjectDBW ) - from gitdb.exc import ( BadObject, AmbiguousObjectName @@ -33,10 +32,8 @@ bin_to_hex, exists, chmod, - isdir, isfile, remove, - mkdir, rename, dirname, basename, @@ -222,8 +219,7 @@ def store(self, istream): if tmp_path: obj_path = self.db_path(self.object_path(hexsha)) obj_dir = dirname(obj_path) - if not isdir(obj_dir): - mkdir(obj_dir) + os.makedirs(obj_dir, exist_ok=True) # END handle destination directory # rename onto existing doesn't work on NTFS if isfile(obj_path): From 9d126ce78e69fb8aba28ae89adde69298a3066a4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 04:33:18 -0400 Subject: [PATCH 0499/1392] Don't cancel other jobs from the 3.12 job failing Because 3.12 is still a release candidate and if tests fail for it then one would always want to know if/how other versions also fail. This also allows actions/setup-python to install a prerelease for 3.12 only, not for other releases. --- .github/workflows/pythonpackage.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index efa04a901..c59e618ac 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -16,6 +16,11 @@ jobs: strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + include: + - experimental: false + - python-version: "3.12" + experimental: true + continue-on-error: ${{ matrix.experimental }} steps: - uses: actions/checkout@v4 @@ -25,7 +30,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - allow-prereleases: true + allow-prereleases: ${{ matrix.experimental }} - name: Install project and dependencies run: | python -m pip install --upgrade pip From 919d3cce57d0a3c398dc4ddbacbbd23942a6ff4c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 04:17:52 -0400 Subject: [PATCH 0500/1392] Use actions/checkout feature to fetch all commits Setting the "fetch-depth" to a positive value fetches that many commits back (and the default value is 1), but setting it to 0 fetches all commits, as in a (deep) normal fetch. --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index efa04a901..14c7ecbb4 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 1000 + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From bd21ed46addf55fea5059d44e0477a785f4a664f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Sep 2023 05:18:27 -0400 Subject: [PATCH 0501/1392] Revert "Drop support for EOL Python 3.7" This brings back Python 3.7 support (allowing it to be installed on 3.7, and testing on 3.7 on CI), even though 3.7 is end-of-life, because support for 3.7 is not being dropped by GitPython yet, and there is value in keeping the version ranges supported by GitPython and gitdb consistent. This reverts commit 875acb45cd8e9353c1911717bd2cd05b3e40ed05. --- .github/workflows/pythonpackage.yml | 2 +- setup.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3d06b4aec..6849763e1 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - experimental: false - python-version: "3.12" diff --git a/setup.py b/setup.py index 57341223b..61b572768 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ zip_safe=False, install_requires=['smmap>=3.0.1,<6'], long_description="""GitDB is a pure-Python git object database""", - python_requires='>=3.8', + python_requires='>=3.7', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 5 - Production/Stable", @@ -35,6 +35,7 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", From 810ae3ad3f94a6a1b54231f88d13b7e071d1278c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Sep 2023 10:22:18 +0200 Subject: [PATCH 0502/1392] prepare v6.0.0 --- doc/source/changes.rst | 7 +++++++ smmap/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index d0623b533..b77de7940 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,13 @@ Changelog ######### +****** +v6.0.0 +****** + +- Dropped support 3.6 and 3.7 +- Declared support for Python 3.11 and 3.12 + ****** v5.0.0 ****** diff --git a/smmap/__init__.py b/smmap/__init__.py index 6b1a44cb5..5f37edae4 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (5, 0, 0) +version_info = (6, 0, 0) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From ce03fde1a0059f6d7db9cf40b9a9d12b75797e7b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Sep 2023 10:25:42 +0200 Subject: [PATCH 0503/1392] improve release setup based on the one in GitPython --- .gitignore | 1 + Makefile | 51 +++++++----------------------------------------- build-release.sh | 26 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 44 deletions(-) create mode 100755 build-release.sh diff --git a/.gitignore b/.gitignore index 01247547d..85cc74fdf 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ MANIFEST *.egg-info .noseids *.sublime-workspace +/env/ diff --git a/Makefile b/Makefile index 051e99ba0..978632e66 100644 --- a/Makefile +++ b/Makefile @@ -1,49 +1,12 @@ -.PHONY: build sdist cover test clean-files clean-docs doc all +.PHONY: all clean release force_release all: - $(info Possible targets:) - $(info doc) - $(info clean-docs) - $(info clean-files) - $(info clean) - $(info test) - $(info coverage) - $(info build) - $(info sdist) + @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all -doc: - cd doc && make html +clean: + rm -rf build/ dist/ .eggs/ .tox/ -clean-docs: - cd doc && make clean - -clean-files: - git clean -fx - rm -rf build/ dist/ - -clean: clean-files clean-docs - -test: - pytest - -coverage: - pytest --cov smmap --cov-report xml - -build: - ./setup.py build - -sdist: - ./setup.py sdist - -release: clean - # Check if latest tag is the current head we're releasing - echo "Latest tag = $$(git tag | sort -nr | head -n1)" - echo "HEAD SHA = $$(git rev-parse head)" - echo "Latest tag SHA = $$(git tag | sort -nr | head -n1 | xargs git rev-parse)" - @test "$$(git rev-parse head)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" - make force_release - -force_release:: clean - git push --tags - python3 setup.py sdist bdist_wheel +force_release: clean + ./build-release.sh twine upload dist/* + git push --tags origin main diff --git a/build-release.sh b/build-release.sh new file mode 100755 index 000000000..5840e4472 --- /dev/null +++ b/build-release.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# This script builds a release. If run in a venv, it auto-installs its tools. +# You may want to run "make release" instead of running this script directly. + +set -eEu + +function release_with() { + $1 -m build --sdist --wheel +} + +if test -n "${VIRTUAL_ENV:-}"; then + deps=(build twine) # Install twine along with build, as we need it later. + echo "Virtual environment detected. Adding packages: ${deps[*]}" + pip install --quiet --upgrade "${deps[@]}" + echo 'Starting the build.' + release_with python +else + function suggest_venv() { + venv_cmd='python -m venv env && source env/bin/activate' + printf "HELP: To avoid this error, use a virtual-env with '%s' instead.\n" "$venv_cmd" + } + trap suggest_venv ERR # This keeps the original exit (error) code. + echo 'Starting the build.' + release_with python3 # Outside a venv, use python3. +fi From e163592304fbe1015605270cfdf4e3529904a986 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Sep 2023 10:30:54 +0200 Subject: [PATCH 0504/1392] adjust force-release target to handle differently named main branch --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 978632e66..20436bbc4 100644 --- a/Makefile +++ b/Makefile @@ -9,4 +9,4 @@ clean: force_release: clean ./build-release.sh twine upload dist/* - git push --tags origin main + git push --tags origin master From b98fdd1fad62c2a957ba76dee5361d55a0cba14c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Sep 2023 04:49:58 -0400 Subject: [PATCH 0505/1392] Revert "Drop support for EOL Python 3.7" Analogous to https://github.com/gitpython-developers/gitdb/pull/94, this brings back Python 3.7 support (allowing it to be installed on 3.7, and testing on 3.7), even though 3.7 is end of life, because support for 3.7 is not being dropped by GitPython (or gitdb) yet, and there is value in keeping GitPython, gitdb, and smmap consistent. This reverts commit f1b4d14e7d3902135af336ed06a12308dfd40270. --- .github/workflows/pythonpackage.yml | 2 +- README.md | 2 +- doc/source/intro.rst | 2 +- setup.py | 3 ++- tox.ini | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 55c654269..73613588d 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index ff973bae6..e21f53453 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ For performance critical 64 bit applications, a simplified version of memory map ## Prerequisites -* Python 3.8+ +* Python 3.7+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 8f36481c2..109fec247 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -20,7 +20,7 @@ For performance critical 64 bit applications, a simplified version of memory map ############# Prerequisites ############# -* Python 3.8+ +* Python 3.7+ * OSX, Windows or Linux The package was tested on all of the previously mentioned configurations. diff --git a/setup.py b/setup.py index b99532814..49d90a02d 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ license="BSD", packages=find_packages(), zip_safe=True, - python_requires=">=3.8", + python_requires=">=3.7", classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -38,6 +38,7 @@ "Operating System :: MacOS :: MacOS X", "Programming Language :: Python", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tox.ini b/tox.ini index ac3536caa..f7205c5a3 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py{38, 39, 310, 311, 312} +envlist = flake8, py{37, 38, 39, 310, 311, 312} [testenv] commands = {envpython} -m pytest --cov smmap --cov-report xml {posargs} From 256c5a21de2d14aca02c9689d7d63f78c4e0ef61 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 17 Sep 2023 13:31:58 +0200 Subject: [PATCH 0506/1392] prepare v5.0.1 --- doc/source/changes.rst | 8 ++++++++ smmap/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index b77de7940..947c71a81 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,10 +2,18 @@ Changelog ######### +****** +v5.0.1 +****** + +- Added support for Python 3.12 + ****** v6.0.0 ****** +YANKED + - Dropped support 3.6 and 3.7 - Declared support for Python 3.11 and 3.12 diff --git a/smmap/__init__.py b/smmap/__init__.py index 5f37edae4..429f47aed 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (6, 0, 0) +version_info = (5, 0, 1) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From e0769d1ed2d9044d7523c2eb2f8a0d44a90deb9e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Sep 2023 08:36:34 -0400 Subject: [PATCH 0507/1392] Fix top-of-file license URLs here in gitdb too This is the gitdb part of the fix for the top-of-file license URLs that have come to point to a page about a related but different license from the one GitPython and gitdb are (intended to be) offered under. See https://github.com/gitpython-developers/GitPython/pull/1662 for details about the problem and how it came about. --- gitdb/__init__.py | 2 +- gitdb/base.py | 2 +- gitdb/db/__init__.py | 2 +- gitdb/db/base.py | 2 +- gitdb/db/git.py | 2 +- gitdb/db/loose.py | 2 +- gitdb/db/mem.py | 2 +- gitdb/db/pack.py | 2 +- gitdb/db/ref.py | 2 +- gitdb/exc.py | 2 +- gitdb/fun.py | 2 +- gitdb/pack.py | 6 +++--- gitdb/stream.py | 10 +++++----- gitdb/test/__init__.py | 2 +- gitdb/test/db/__init__.py | 2 +- gitdb/test/db/lib.py | 2 +- gitdb/test/db/test_git.py | 2 +- gitdb/test/db/test_loose.py | 2 +- gitdb/test/db/test_mem.py | 2 +- gitdb/test/db/test_pack.py | 2 +- gitdb/test/db/test_ref.py | 2 +- gitdb/test/lib.py | 2 +- gitdb/test/performance/lib.py | 2 +- gitdb/test/performance/test_pack.py | 2 +- gitdb/test/performance/test_pack_streaming.py | 2 +- gitdb/test/performance/test_stream.py | 2 +- gitdb/test/test_base.py | 2 +- gitdb/test/test_example.py | 2 +- gitdb/test/test_pack.py | 4 ++-- gitdb/test/test_stream.py | 2 +- gitdb/test/test_util.py | 2 +- gitdb/typ.py | 2 +- gitdb/util.py | 2 +- 33 files changed, 40 insertions(+), 40 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index b777632bb..2fb3f7edb 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Initialize the object database module""" import sys diff --git a/gitdb/base.py b/gitdb/base.py index 42e71d0fa..9a23a4f71 100644 --- a/gitdb/base.py +++ b/gitdb/base.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module with basic data structures - they are designed to be lightweight and fast""" from gitdb.util import bin_to_hex diff --git a/gitdb/db/__init__.py b/gitdb/db/__init__.py index 0a2a46a64..20fd2280d 100644 --- a/gitdb/db/__init__.py +++ b/gitdb/db/__init__.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.db.base import * from gitdb.db.loose import * diff --git a/gitdb/db/base.py b/gitdb/db/base.py index e89052e88..7312fe00f 100644 --- a/gitdb/db/base.py +++ b/gitdb/db/base.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains implementations of database retrieveing objects""" from gitdb.util import ( join, diff --git a/gitdb/db/git.py b/gitdb/db/git.py index e2cb46805..a1ed14280 100644 --- a/gitdb/db/git.py +++ b/gitdb/db/git.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.db.base import ( CompoundDB, ObjectDBW, diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 7ea6fef3d..256fec9cf 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.db.base import ( FileDBBase, ObjectDBR, diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py index 1b954cba2..d4772fdb5 100644 --- a/gitdb/db/mem.py +++ b/gitdb/db/mem.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains the MemoryDatabase implementation""" from gitdb.db.loose import LooseObjectDB from gitdb.db.base import ( diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py index 1ce786b79..274ea5990 100644 --- a/gitdb/db/pack.py +++ b/gitdb/db/pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module containing a database to deal with packs""" from gitdb.db.base import ( FileDBBase, diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index 6bb2a64dd..bd3015602 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ import codecs from gitdb.db.base import ( CompoundDB, diff --git a/gitdb/exc.py b/gitdb/exc.py index 947e5d8bf..73970372b 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module with common exceptions""" from gitdb.util import to_hex_sha diff --git a/gitdb/fun.py b/gitdb/fun.py index a4454de4d..a272e5caa 100644 --- a/gitdb/fun.py +++ b/gitdb/fun.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains basic c-functions which usually contain performance critical code Keeping this code separate from the beginning makes it easier to out-source it into c later, if required""" diff --git a/gitdb/pack.py b/gitdb/pack.py index cabce4c9d..e559e113d 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains PackIndexFile and PackFile implementations""" import zlib @@ -263,7 +263,7 @@ def __init__(self, indexpath): def close(self): mman.force_map_handle_removal_win(self._indexpath) self._cursor = None - + def _set_cache_(self, attr): if attr == "_packfile_checksum": self._packfile_checksum = self._cursor.map()[-40:-20] @@ -528,7 +528,7 @@ def __init__(self, packpath): def close(self): mman.force_map_handle_removal_win(self._packpath) self._cursor = None - + def _set_cache_(self, attr): # we fill the whole cache, whichever attribute gets queried first self._cursor = mman.make_cursor(self._packpath).use_region() diff --git a/gitdb/stream.py b/gitdb/stream.py index 1b5426f02..1e0be84fd 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from io import BytesIO @@ -140,7 +140,7 @@ def data(self): def close(self): """Close our underlying stream of compressed bytes if this was allowed during initialization :return: True if we closed the underlying stream - :note: can be called safely + :note: can be called safely """ if self._close: if hasattr(self._m, 'close'): @@ -287,11 +287,11 @@ def read(self, size=-1): # if we hit the end of the stream # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. # They are thorough, and I assume it is truly working. - # Why is this logic as convoluted as it is ? Please look at the table in + # Why is this logic as convoluted as it is ? Please look at the table in # https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. # Basically, on py2.6, you want to use branch 1, whereas on all other python version, the second branch - # will be the one that works. - # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the + # will be the one that works. + # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the # table in the github issue. This is it ... it was the only way I could make this work everywhere. # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . if getattr(zlib, 'ZLIB_RUNTIME_VERSION', zlib.ZLIB_VERSION) in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py index 8a681e428..03bd406be 100644 --- a/gitdb/test/__init__.py +++ b/gitdb/test/__init__.py @@ -1,4 +1,4 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/gitdb/test/db/__init__.py b/gitdb/test/db/__init__.py index 8a681e428..03bd406be 100644 --- a/gitdb/test/db/__init__.py +++ b/gitdb/test/db/__init__.py @@ -1,4 +1,4 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py index b38f1d5e5..408dd8c66 100644 --- a/gitdb/test/db/lib.py +++ b/gitdb/test/db/lib.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Base classes for object db testing""" from gitdb.test.lib import ( with_rw_directory, diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py index 6ecf7d7a8..73ac1a08c 100644 --- a/gitdb/test/db/test_git.py +++ b/gitdb/test/db/test_git.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ import os from gitdb.test.db.lib import ( TestDBBase, diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py index 8cc660b8a..295e2eecd 100644 --- a/gitdb/test/db/test_loose.py +++ b/gitdb/test/db/test_loose.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory diff --git a/gitdb/test/db/test_mem.py b/gitdb/test/db/test_mem.py index eb563c098..882e54fe5 100644 --- a/gitdb/test/db/test_mem.py +++ b/gitdb/test/db/test_mem.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory diff --git a/gitdb/test/db/test_pack.py b/gitdb/test/db/test_pack.py index 4539f4278..bd07906da 100644 --- a/gitdb/test/db/test_pack.py +++ b/gitdb/test/db/test_pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory, diff --git a/gitdb/test/db/test_ref.py b/gitdb/test/db/test_ref.py index 664aa54ba..0816e649f 100644 --- a/gitdb/test/db/test_ref.py +++ b/gitdb/test/db/test_ref.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ from gitdb.test.db.lib import ( TestDBBase, with_rw_directory, diff --git a/gitdb/test/lib.py b/gitdb/test/lib.py index da59d3b35..8e602342c 100644 --- a/gitdb/test/lib.py +++ b/gitdb/test/lib.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Utilities used in ODB testing""" from gitdb import OStream diff --git a/gitdb/test/performance/lib.py b/gitdb/test/performance/lib.py index fa4dd209d..36916ede3 100644 --- a/gitdb/test/performance/lib.py +++ b/gitdb/test/performance/lib.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Contains library functions""" from gitdb.test.lib import TestBase diff --git a/gitdb/test/performance/test_pack.py b/gitdb/test/performance/test_pack.py index f034baffd..fc3d3349f 100644 --- a/gitdb/test/performance/test_pack.py +++ b/gitdb/test/performance/test_pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Performance tests for object store""" from gitdb.test.performance.lib import ( diff --git a/gitdb/test/performance/test_pack_streaming.py b/gitdb/test/performance/test_pack_streaming.py index db790f1db..80c798bbc 100644 --- a/gitdb/test/performance/test_pack_streaming.py +++ b/gitdb/test/performance/test_pack_streaming.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Specific test for pack streams only""" from gitdb.test.performance.lib import ( diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py index 91dc89178..fb10871ae 100644 --- a/gitdb/test/performance/test_stream.py +++ b/gitdb/test/performance/test_stream.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Performance data streaming performance""" from gitdb.test.performance.lib import TestBigRepoR diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 519cdfdc1..8fc9e35bf 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Test for object db""" from gitdb.test.lib import ( TestBase, diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py index cc4d40dd3..3b4c9084b 100644 --- a/gitdb/test/test_example.py +++ b/gitdb/test/test_example.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module with examples from the tutorial section of the docs""" import os from gitdb.test.lib import TestBase diff --git a/gitdb/test/test_pack.py b/gitdb/test/test_pack.py index f9461975b..e72348228 100644 --- a/gitdb/test/test_pack.py +++ b/gitdb/test/test_pack.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Test everything about packs reading and writing""" from gitdb.test.lib import ( TestBase, @@ -242,7 +242,7 @@ def rewind_streams(): # END for each info assert count == len(pack_objs) entity.close() - + def test_pack_64(self): # TODO: hex-edit a pack helping us to verify that we can handle 64 byte offsets # of course without really needing such a huge pack diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 5e2e1bab2..1e7e941d1 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Test for object db""" from gitdb.test.lib import ( diff --git a/gitdb/test/test_util.py b/gitdb/test/test_util.py index 3b3165d14..166b33c3a 100644 --- a/gitdb/test/test_util.py +++ b/gitdb/test/test_util.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Test for object db""" import tempfile import os diff --git a/gitdb/typ.py b/gitdb/typ.py index 98d15f3ec..314db50a7 100644 --- a/gitdb/typ.py +++ b/gitdb/typ.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ """Module containing information about types known to the database""" str_blob_type = b'blob' diff --git a/gitdb/util.py b/gitdb/util.py index 3151c061e..bb6d8797a 100644 --- a/gitdb/util.py +++ b/gitdb/util.py @@ -1,7 +1,7 @@ # Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors # # This module is part of GitDB and is released under -# the New BSD License: http://www.opensource.org/licenses/bsd-license.php +# the New BSD License: https://opensource.org/license/bsd-3-clause/ import binascii import os import mmap From f1ddf0155ce22041b2db19cdf902e30e0db97c54 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Sep 2023 08:55:21 -0400 Subject: [PATCH 0508/1392] Update ci, in line with gitdb This updates smmap's CI configuration in ways that are in line with recent updates to gitdb's. In most cases there is no difference in the changes, and the reason for the updates is more to avoid confusing differences than from the value of the changes themselves. In one case, there is a major difference (fetch-depth). - https://github.com/gitpython-developers/gitdb/pull/89 (same) - https://github.com/gitpython-developers/gitdb/pull/90 (same) It's just the project, not dependencies, but otherwise the same. - https://github.com/gitpython-developers/gitdb/pull/92 (opposite) This is the major difference. We don't need more than the tip of the branch in these tests. Keeping the default fetch-depth of 1 by not setting it explicitly avoids giving the impression that the tests here are doing something they are not (and also serves as a speed optimization). - https://github.com/gitpython-developers/gitdb/pull/93 (same) --- .github/dependabot.yml | 6 ++++++ .github/workflows/pythonpackage.yml | 12 ++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..203f3c889 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 73613588d..7e21bb9a7 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -16,19 +16,23 @@ jobs: strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + include: + - experimental: false + - python-version: "3.12" + experimental: true + continue-on-error: ${{ matrix.experimental }} steps: - uses: actions/checkout@v4 - with: - fetch-depth: 1000 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - allow-prereleases: true - - name: Install dependencies + allow-prereleases: ${{ matrix.experimental }} + - name: Install project run: | python -m pip install --upgrade pip + pip install . - name: Lint with flake8 run: | pip install flake8 From cd8da415e119451e195903576a8ff22cf60ae47b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 12:42:23 -0400 Subject: [PATCH 0509/1392] Run CI on all branches This makes it easier to test changes to CI without/before a PR. + Fix a small YAML indentation style inconsistency. --- .github/workflows/pythonpackage.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 6849763e1..6d819ff42 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -3,11 +3,7 @@ name: Python package -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] +on: [push, pull_request, workflow_dispatch] jobs: build: @@ -17,9 +13,9 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - - experimental: false - - python-version: "3.12" - experimental: true + - experimental: false + - python-version: "3.12" + experimental: true continue-on-error: ${{ matrix.experimental }} steps: From 00bbc44eff7be0b43a882eadf54c2bbb2f9000a0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 12:44:38 -0400 Subject: [PATCH 0510/1392] No longer treat 3.12 as experimental on CI Since Python 3.12.0 stable has been released, as well as now being available via setup-python, per: https://github.com/actions/python-versions/blob/main/versions-manifest.json The main practical effect of this is that continue-on-error is no longer set to true for 3.12, so 3.12 is no longer special-cased to refrain from cancelling other test jobs when its test job fails. Another effect is that 3.12 can longer be selected as a prerelease. --- .github/workflows/pythonpackage.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 6d819ff42..73b3902c3 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -14,8 +14,6 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - experimental: false - - python-version: "3.12" - experimental: true continue-on-error: ${{ matrix.experimental }} steps: From 69cd90ff968bd466b26d52859e5a850b6ce300b1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 13:09:45 -0400 Subject: [PATCH 0511/1392] Run CI on all branches + Make YAML indentation more consistent. --- .github/workflows/pythonpackage.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 7e21bb9a7..28054ea52 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -3,11 +3,7 @@ name: Python package -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] +on: [push, pull_request, workflow_dispatch] jobs: build: @@ -17,9 +13,9 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - - experimental: false - - python-version: "3.12" - experimental: true + - experimental: false + - python-version: "3.12" + experimental: true continue-on-error: ${{ matrix.experimental }} steps: From 44ac30a7481a619e32b13a4efc1de0479b1b4379 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 3 Oct 2023 13:11:57 -0400 Subject: [PATCH 0512/1392] No longer treat 3.12 as experimental on CI Since Python 3.12.0 stable has been released and is available via setup-python, per: https://github.com/actions/python-versions/blob/main/versions-manifest.json This makes it so continue-on-error and allow-prereleases are no longer special-cased to true for 3.12; instead, 3.12 is treaed the same as other Python releases (where those are false). --- .github/workflows/pythonpackage.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 28054ea52..e816355f3 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -14,8 +14,6 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] include: - experimental: false - - python-version: "3.12" - experimental: true continue-on-error: ${{ matrix.experimental }} steps: From 70098c9c99db872ac98a6b5c7a591a42cadf049f Mon Sep 17 00:00:00 2001 From: DeflateAwning <11021263+DeflateAwning@users.noreply.github.com> Date: Fri, 6 Oct 2023 12:22:44 -0600 Subject: [PATCH 0513/1392] Add __all__ to exc for linting --- gitdb/exc.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/gitdb/exc.py b/gitdb/exc.py index 73970372b..752dafdbb 100644 --- a/gitdb/exc.py +++ b/gitdb/exc.py @@ -5,6 +5,17 @@ """Module with common exceptions""" from gitdb.util import to_hex_sha +__all__ = [ + 'AmbiguousObjectName', + 'BadName', + 'BadObject', + 'BadObjectType', + 'InvalidDBRoot', + 'ODBError', + 'ParseError', + 'UnsupportedOperation', + 'to_hex_sha', +] class ODBError(Exception): """All errors thrown by the object database""" From 12db86cefa90e8fc1c8df8f5c6b4c8b7a232d773 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 13 Oct 2023 04:06:47 -0400 Subject: [PATCH 0514/1392] Have Dependabot update smmap submodule dependency This makes Dependabot open version update PRs for submodules (which here is just smmap), as well as GitHub Actions. This is like https://github.com/gitpython-developers/GitPython/pull/1702. --- .github/dependabot.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 203f3c889..5acde1a9a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,4 +3,9 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: "weekly" + +- package-ecosystem: "gitsubmodule" + directory: "/" + schedule: + interval: "monthly" From 8e613962153baad0431358dda03d9fd942cef616 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 08:17:45 +0000 Subject: [PATCH 0515/1392] Bump gitdb/ext/smmap from `334ef84` to `f1ace75` Bumps [gitdb/ext/smmap](https://github.com/gitpython-developers/smmap) from `334ef84` to `f1ace75`. - [Commits](https://github.com/gitpython-developers/smmap/compare/334ef84a05c953ed5dbec7b9c6d4310879eeab5a...f1ace75be355fdec927793e462b9b12bf6ec9520) --- updated-dependencies: - dependency-name: gitdb/ext/smmap dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- gitdb/ext/smmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 334ef84a0..f1ace75be 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 334ef84a05c953ed5dbec7b9c6d4310879eeab5a +Subproject commit f1ace75be355fdec927793e462b9b12bf6ec9520 From 2057ae67b85fb9925efbd0f00f44413e506e286c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 20 Oct 2023 09:37:58 +0200 Subject: [PATCH 0516/1392] bump versions to prepare for next release --- doc/source/changes.rst | 6 ++++++ gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 7d85ea55b..0b8de13d5 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +****** +4.0.11 +****** + +- various improvements - please see the release on GitHub for details. + ****** 4.0.10 ****** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 2fb3f7edb..803a4283f 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -28,7 +28,7 @@ def _init_externals(): __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 10) +version_info = (4, 0, 11) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index f1ace75be..256c5a21d 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit f1ace75be355fdec927793e462b9b12bf6ec9520 +Subproject commit 256c5a21de2d14aca02c9689d7d63f78c4e0ef61 diff --git a/setup.py b/setup.py index 61b572768..f67f7a58f 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 10) +version_info = (4, 0, 11) __version__ = '.'.join(str(i) for i in version_info) setup( From 3d3e9572dc452fea53d328c101b3d1440bbefe40 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 20 Oct 2023 09:42:19 +0200 Subject: [PATCH 0517/1392] fix makefile to allow building packages with current python version It all breaks all the time, it's just like that. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7aa5a717a..a0a2d0e01 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ release:: clean force_release:: clean git push --tags - python3 setup.py sdist bdist_wheel + python3 -m build --sdist --wheel twine upload dist/* doc:: From 965d2d36703a60f610e4b4a3fb1b86fe244d63d5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 20 Oct 2023 06:46:10 -0400 Subject: [PATCH 0518/1392] Never add a vendored smmap directory to sys.path This removes the logic that appended the git submodule directory for smmap to sys.path under most circumstances when the version of gitdb was not from PyPI. Now gitdb does not modify sys.path. See https://github.com/gitpython-developers/GitPython/issues/1717 and https://github.com/gitpython-developers/GitPython/pull/1720 for context. This change is roughly equivalent to the change to GitPython, though as noted the behavior being eliminated is subtly different here and there. --- gitdb/__init__.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 803a4283f..9b77e9f2d 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -4,34 +4,12 @@ # the New BSD License: https://opensource.org/license/bsd-3-clause/ """Initialize the object database module""" -import sys -import os - -#{ Initialization - - -def _init_externals(): - """Initialize external projects by putting them into the path""" - if 'PYOXIDIZER' not in os.environ: - where = os.path.join(os.path.dirname(__file__), 'ext', 'smmap') - if os.path.exists(where): - sys.path.append(where) - - import smmap - del smmap - # END handle imports - -#} END initialization - -_init_externals() - __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" version_info = (4, 0, 11) __version__ = '.'.join(str(i) for i in version_info) - # default imports from gitdb.base import * from gitdb.db import * From dfbfb12beee6b2c61cb02f193fabc427e0a949f6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 20 Oct 2023 07:24:33 -0400 Subject: [PATCH 0519/1392] Revise and update the readme Changes worth mentioning: - Format commands as code blocks instead of blockquotes. (This is particularly useful for the submodule update step, whose lines were inadvertently concatenated, but it also improves appearance overall.) - Mention smmap as a requirement. (But also that it doesn't need to be separately installed.) - Mention that gitdb-speedups is not currently maintained. - No longer say gitdb has source code in gitorious. (Since that site no longer exists.) - Call GitPython "GitPython" rather than "git-python". - Replace the old git-python Google Groups link with a link to the Discussions page on the GitHub repository for GitPython. (This seems like the closest currently available resource.) --- README.rst | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 29c70f781..61ce28b1e 100644 --- a/README.rst +++ b/README.rst @@ -16,34 +16,38 @@ Installation :target: https://readthedocs.org/projects/gitdb/?badge=latest :alt: Documentation Status -From `PyPI `_ +From `PyPI `_:: pip install gitdb SPEEDUPS ======== -If you want to go up to 20% faster, you can install gitdb-speedups with: +If you want to go up to 20% faster, you can install gitdb-speedups with:: pip install gitdb-speedups +However, please note that gitdb-speedups is not currently maintained. + REQUIREMENTS ============ +* smmap - declared as a dependency, automatically installed * pytest - for running the tests SOURCE ====== -The source is available in a git repository at gitorious and github: + +The source is available in a git repository on GitHub: https://github.com/gitpython-developers/gitdb -Once the clone is complete, please be sure to initialize the submodules using +Once the clone is complete, please be sure to initialize the submodule using:: cd gitdb git submodule update --init -Run the tests with +Run the tests with:: pytest @@ -53,13 +57,13 @@ DEVELOPMENT .. image:: https://github.com/gitpython-developers/gitdb/workflows/Python%20package/badge.svg :target: https://github.com/gitpython-developers/gitdb/actions -The library is considered mature, and not under active development. It's primary (known) use is in git-python. +The library is considered mature, and not under active development. Its primary (known) use is in GitPython. INFRASTRUCTURE ============== -* Mailing List - * http://groups.google.com/group/git-python +* Discussions + * https://github.com/gitpython-developers/GitPython/discussions * Issue Tracker * https://github.com/gitpython-developers/gitdb/issues From e998429c01f928da7ff7c922ba3f1249c43ff569 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 20 Oct 2023 07:47:10 -0400 Subject: [PATCH 0520/1392] Set Dependabot submodule update cadence to weekly This changes it from monthly to weekly. See #99 and https://github.com/gitpython-developers/GitPython/pull/1702#issuecomment-1761182333 for context. --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5acde1a9a..2fe73ca77 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,4 +8,4 @@ updates: - package-ecosystem: "gitsubmodule" directory: "/" schedule: - interval: "monthly" + interval: "weekly" From 24ecf58262eb2e76906689dbc4e28397f4f628dc Mon Sep 17 00:00:00 2001 From: Antoine C Date: Fri, 8 Dec 2023 16:58:24 +0100 Subject: [PATCH 0521/1392] fix #101 --- gitdb/test/test_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitdb/test/test_base.py b/gitdb/test/test_base.py index 8fc9e35bf..17906c9c2 100644 --- a/gitdb/test/test_base.py +++ b/gitdb/test/test_base.py @@ -73,7 +73,7 @@ def test_streams(self): # test deltapackstream dpostream = ODeltaPackStream(*(dpinfo + (stream, ))) - dpostream.stream is stream + assert dpostream.stream is stream dpostream.read(5) stream._assert() assert stream.bytes == 5 @@ -92,7 +92,7 @@ def test_streams(self): assert istream.size == s istream.size = s * 2 - istream.size == s * 2 + assert istream.size == s * 2 assert istream.type == str_blob_type istream.type = "something" assert istream.type == "something" From 36dd418b05f962569fd894d4edf533fc8d7c5bed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 02:10:45 +0000 Subject: [PATCH 0522/1392] Bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index e816355f3..fd73d6dd6 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -19,7 +19,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} From 86402e67e7d999b2b2665dc1029c5e1ccd3ada35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 11:03:07 +0000 Subject: [PATCH 0523/1392] Bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 73b3902c3..ec7550de2 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -21,7 +21,7 @@ jobs: with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} From 41fac851fb62b4224e0400be46a3884708d185c7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Dec 2023 19:02:14 -0500 Subject: [PATCH 0524/1392] Avoid mktemp in tests, in straightforward cases The tempfile.mktemp function is deprecated, because of a race condition where the file may be concurrently created between when its name is generated and when it is opened. Other faciliies in the tempfile module overcome this by generating a name, attempting to create the file or directory in a way that guarantees failure if it already existed, and, in the occasional case that it did already exist, generating another name and trying again (stopping after a predefined limit). For further information on mktemp deprecation: - https://docs.python.org/3/library/tempfile.html#tempfile.mktemp - https://github.com/gitpython-developers/smmap/issues/41 The security risk of calls to mktemp in this project's test suite is low. However, it is still best to avoid using it, because it is deprecated, because it is (at least slightly) brittle, and because any use of mktemp looks like a potential security risk and thereby imposes a burden on working with the code (which could potentially be addressed with detailed comments analyzing why it is believed safe in particular cases, but this would typically be more verbose, and at least as challenging to add, as replacing mktemp with a better alternative). This commit replaces *some* uses of mktemp in the test suite: those where it is readily clear how to do so in a way that preserves the code's intent: - Where a name for a temporary directory is generated with mktemp and os.mkdir is called immediately, mkdtemp is now used. - Where a name for a temporary file that is not customized (such as with a prefix) is generated with mktemp, such that the code under test never uses the filename but only the already-open file-like object, TemporaryFile is now used. As the name isn't customized, the test code in these cases does not express an intent to allow the developer to inspect the file after a test failure, so even if the file wasn't guaranteed to be deleted with a finally block or context manager, it is fine to do so. TemporaryFile supports this use case well on all systems including Windows, and automatically deletes the file. - Where a name for a temporary file that *is* customized (such as with a prefix) to reflect the way the test uses it is generated with mktemp, and the test code does not attempt deterministic deletion of the file when an exception would make the test fail, NamedTemporaryFile with delete=False is now used. The original code to remove the file when the test succeeds is modified accordingly to do the same job, and also commented to explain that it is being handled this way to allow the file to be kept and examined when a test failure occurs. Other cases in the test suite should also be feasible to replace, but are left alone for now. Some of them are ambiguous in their intent, with respect to whether the file should be retained after a test failure. Others appear deliberately to avoid creating a file or directory where the code under test should do so, possibly to check that this is done properly. (One way to preserve that latter behavior, while avoiding the weakness of using mktemp and also avoiding inadverently reproducing that weakness by other means, could be to use a path in a temporary directory made for the test.) This commit also doesn't address the one use of mktemp in the code under test (i.e., outside the test suite, inside the git module). --- test/lib/helper.py | 3 +- test/performance/lib.py | 3 +- test/test_base.py | 5 ++-- test/test_reflog.py | 7 ++--- test/test_repo.py | 5 ++-- test/test_util.py | 64 ++++++++++++++++++++++------------------- 6 files changed, 42 insertions(+), 45 deletions(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index 58d96534a..d662b632d 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -89,8 +89,7 @@ def with_rw_directory(func): @wraps(func) def wrapper(self): - path = tempfile.mktemp(prefix=func.__name__) - os.mkdir(path) + path = tempfile.mkdtemp(prefix=func.__name__) keep = False try: return func(self, path) diff --git a/test/performance/lib.py b/test/performance/lib.py index ceee6c2a1..d08e1027f 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -65,8 +65,7 @@ class TestBigRepoRW(TestBigRepoR): def setUp(self): self.gitrwrepo = None super().setUp() - dirname = tempfile.mktemp() - os.mkdir(dirname) + dirname = tempfile.mkdtemp() self.gitrwrepo = self.gitrorepo.clone(dirname, shared=True, bare=True, odbt=GitCmdObjectDB) self.puregitrwrepo = Repo(dirname, odbt=GitDB) diff --git a/test/test_base.py b/test/test_base.py index cdf82b74d..74f342071 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -68,12 +68,11 @@ def test_base_object(self): data = data_stream.read() assert data - tmpfilename = tempfile.mktemp(suffix="test-stream") - with open(tmpfilename, "wb+") as tmpfile: + with tempfile.NamedTemporaryFile(suffix="test-stream", delete=False) as tmpfile: self.assertEqual(item, item.stream_data(tmpfile)) tmpfile.seek(0) self.assertEqual(tmpfile.read(), data) - os.remove(tmpfilename) + os.remove(tmpfile.name) # Do it this way so we can inspect the file on failure. # END for each object type to create # Each has a unique sha. diff --git a/test/test_reflog.py b/test/test_reflog.py index 625466d40..1bd2e5dab 100644 --- a/test/test_reflog.py +++ b/test/test_reflog.py @@ -1,7 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -import os +import os.path as osp import tempfile from git.objects import IndexObject @@ -9,8 +9,6 @@ from test.lib import TestBase, fixture_path from git.util import Actor, rmtree, hex_to_bin -import os.path as osp - class TestRefLog(TestBase): def test_reflogentry(self): @@ -35,8 +33,7 @@ def test_reflogentry(self): def test_base(self): rlp_head = fixture_path("reflog_HEAD") rlp_master = fixture_path("reflog_master") - tdir = tempfile.mktemp(suffix="test_reflogs") - os.mkdir(tdir) + tdir = tempfile.mkdtemp(suffix="test_reflogs") rlp_master_ro = RefLog.path(self.rorepo.head) assert osp.isfile(rlp_master_ro) diff --git a/test/test_repo.py b/test/test_repo.py index 007b8ecb0..465bb2574 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -667,11 +667,10 @@ def test_tag_to_full_tag_path(self): self.assertEqual(value_errors, []) def test_archive(self): - tmpfile = tempfile.mktemp(suffix="archive-test") - with open(tmpfile, "wb") as stream: + with tempfile.NamedTemporaryFile("wb", suffix="archive-test", delete=False) as stream: self.rorepo.archive(stream, "0.1.6", path="doc") assert stream.tell() - os.remove(tmpfile) + os.remove(stream.name) # Do it this way so we can inspect the file on failure. @mock.patch.object(Git, "_call_process") def test_should_display_blame_information(self, git): diff --git a/test/test_util.py b/test/test_util.py index 1cb255cfe..6616e1067 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -359,48 +359,52 @@ def test_it_should_dashify(self): self.assertEqual("foo", dashify("foo")) def test_lock_file(self): - my_file = tempfile.mktemp() - lock_file = LockFile(my_file) - assert not lock_file._has_lock() - # Release lock we don't have - fine. - lock_file._release_lock() + with tempfile.TemporaryDirectory() as tdir: + my_file = os.path.join(tdir, "my-lock-file") + lock_file = LockFile(my_file) + assert not lock_file._has_lock() + # Release lock we don't have - fine. + lock_file._release_lock() - # Get lock. - lock_file._obtain_lock_or_raise() - assert lock_file._has_lock() + # Get lock. + lock_file._obtain_lock_or_raise() + assert lock_file._has_lock() - # Concurrent access. - other_lock_file = LockFile(my_file) - assert not other_lock_file._has_lock() - self.assertRaises(IOError, other_lock_file._obtain_lock_or_raise) + # Concurrent access. + other_lock_file = LockFile(my_file) + assert not other_lock_file._has_lock() + self.assertRaises(IOError, other_lock_file._obtain_lock_or_raise) - lock_file._release_lock() - assert not lock_file._has_lock() + lock_file._release_lock() + assert not lock_file._has_lock() - other_lock_file._obtain_lock_or_raise() - self.assertRaises(IOError, lock_file._obtain_lock_or_raise) + other_lock_file._obtain_lock_or_raise() + self.assertRaises(IOError, lock_file._obtain_lock_or_raise) - # Auto-release on destruction. - del other_lock_file - lock_file._obtain_lock_or_raise() - lock_file._release_lock() + # Auto-release on destruction. + del other_lock_file + lock_file._obtain_lock_or_raise() + lock_file._release_lock() def test_blocking_lock_file(self): - my_file = tempfile.mktemp() - lock_file = BlockingLockFile(my_file) - lock_file._obtain_lock() - - # Next one waits for the lock. - start = time.time() - wait_time = 0.1 - wait_lock = BlockingLockFile(my_file, 0.05, wait_time) - self.assertRaises(IOError, wait_lock._obtain_lock) - elapsed = time.time() - start + with tempfile.TemporaryDirectory() as tdir: + my_file = os.path.join(tdir, "my-lock-file") + lock_file = BlockingLockFile(my_file) + lock_file._obtain_lock() + + # Next one waits for the lock. + start = time.time() + wait_time = 0.1 + wait_lock = BlockingLockFile(my_file, 0.05, wait_time) + self.assertRaises(IOError, wait_lock._obtain_lock) + elapsed = time.time() - start + extra_time = 0.02 if os.name == "nt" or sys.platform == "cygwin": extra_time *= 6 # Without this, we get indeterministic failures on Windows. elif sys.platform == "darwin": extra_time *= 18 # The situation on macOS is similar, but with more delay. + self.assertLess(elapsed, wait_time + extra_time) def test_user_id(self): From 9e86053c885a27360d9d4699fafaef5e995d6ec5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Dec 2023 22:30:51 -0500 Subject: [PATCH 0525/1392] Replace the one use of mktemp in the git module This makes two related changes to git.index.util.TemporaryFileSwap: - Replace mktemp with mkstemp and then immediately closing the file. This avoids two possible name clashes: the usual name clash where the file may be created by another process between when mktemp generates the name and when the file is created, and the problem that mktemp does not check for files that contain the generated name as a part. The latter is specific to the use here, where a string generated by mktemp was manually concatenated to the base filename. This addresses that by passing the filename as the prefix for mkstemp to use. - Replace os.remove with os.replace and simplify. This is made necessary on Windows by the file already existing, due to mkstemp creating it. Deleting the file and allowing it to be recreated would reproduce the mktemp race condition in full (obscured and with extra steps). But os.replace supports an existing target file on all platforms. It is now also used in __exit__, where it allows the Windows check for the file to be removed, and (in any OS) better expresses the intent of the code to human readers. Furthermore, because one of the "look before you leap" checks in __exit__ is removed, the minor race condition in cleaning up the file is somewhat decreased. A small amount of related refactoring is included. The interface is not changed, even where it could be simplified (by letting __exit__ return None), and resource acquisition remains done on construction rather than in __enter__, because changing those aspects of the design could break some existing uses. Although the use of mktemp replaced here was in the git module and not the test suite, its use was to generate filenames for use in a directory that would ordinarily be under the user's control, such that the ability to carry out typical mktemp-related attacks would already require the ability to achieve the same goals more easily and reliably. Although TemporaryFileSwap is a public class that may be used directly by applications, it is only useful for making a temporary file in the same directory as an existing file. Thus the intended benefits of this change are mainly to code quality and a slight improvement in robustness. --- git/index/util.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/git/index/util.py b/git/index/util.py index b1aaa58fd..1c3b1c4ad 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -3,6 +3,7 @@ """Index utilities.""" +import contextlib from functools import wraps import os import os.path as osp @@ -40,12 +41,10 @@ class TemporaryFileSwap: def __init__(self, file_path: PathLike) -> None: self.file_path = file_path - self.tmp_file_path = str(self.file_path) + tempfile.mktemp("", "", "") - # It may be that the source does not exist. - try: - os.rename(self.file_path, self.tmp_file_path) - except OSError: - pass + fd, self.tmp_file_path = tempfile.mkstemp(prefix=self.file_path, dir="") + os.close(fd) + with contextlib.suppress(OSError): # It may be that the source does not exist. + os.replace(self.file_path, self.tmp_file_path) def __enter__(self) -> "TemporaryFileSwap": return self @@ -57,10 +56,7 @@ def __exit__( exc_tb: Optional[TracebackType], ) -> bool: if osp.isfile(self.tmp_file_path): - if os.name == "nt" and osp.exists(self.file_path): - os.remove(self.file_path) - os.rename(self.tmp_file_path, self.file_path) - + os.replace(self.tmp_file_path, self.file_path) return False From 83b7ec61a1596971d4834e3c93a51b6efdf9e766 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Dec 2023 03:40:28 -0500 Subject: [PATCH 0526/1392] Set up CodeQL This is an automatically generated CodeQL workflow for the project. It is not yet customized. --- .github/workflows/codeql.yml | 81 ++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..7f64e35cb --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,81 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '27 10 * * 3' + +jobs: + analyze: + name: Analyze + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] + # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" From 58547d82fa36165f611563c6bc449fde54dd3b2e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Dec 2023 03:41:16 -0500 Subject: [PATCH 0527/1392] Customize CodeQL - Run CodeQL on all branches. - Don't install Python dependencies. - Use v4 of actions/checkout. --- .github/workflows/codeql.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7f64e35cb..0d4d81efd 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,9 +13,7 @@ name: "CodeQL" on: push: - branches: [ "main" ] pull_request: - branches: [ "main" ] schedule: - cron: '27 10 * * 3' @@ -45,13 +43,14 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} + setup-python-dependencies: false # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. From a315725c68295a02f3d007409ea3101430543289 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Dec 2023 03:04:54 -0500 Subject: [PATCH 0528/1392] Replace use of mktemp This uses NamedTemporaryFile with delete=False to replace the one use of the deprecated mktemp function in smmap (reported in #41). This avoids the race condition inherent to mktemp, as the file is named and created together in a way that is effectively atomic. Because NamedTemporaryFile is not being used to automatically delete the file, it use and cleanup are unaffected by the change. --- smmap/test/lib.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/smmap/test/lib.py b/smmap/test/lib.py index ca91ee914..b15b0ec6a 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -18,12 +18,12 @@ class FileCreator: def __init__(self, size, prefix=''): assert size, "Require size to be larger 0" - self._path = tempfile.mktemp(prefix=prefix) self._size = size - with open(self._path, "wb") as fp: - fp.seek(size - 1) - fp.write(b'1') + with tempfile.NamedTemporaryFile("wb", prefix=prefix, delete=False) as file: + self._path = file.name + file.seek(size - 1) + file.write(b'1') assert os.path.getsize(self.path) == size From 5aeb6e073ebe007d43695976eccdcca64568d025 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 10:39:28 +0000 Subject: [PATCH 0529/1392] Bump gitdb/ext/smmap from `256c5a2` to `04dd210` Bumps [gitdb/ext/smmap](https://github.com/gitpython-developers/smmap) from `256c5a2` to `04dd210`. - [Commits](https://github.com/gitpython-developers/smmap/compare/256c5a21de2d14aca02c9689d7d63f78c4e0ef61...04dd2103ee6e0b7483889e5feda25053c6df2b52) --- updated-dependencies: - dependency-name: gitdb/ext/smmap dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- gitdb/ext/smmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 256c5a21d..04dd2103e 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 256c5a21de2d14aca02c9689d7d63f78c4e0ef61 +Subproject commit 04dd2103ee6e0b7483889e5feda25053c6df2b52 From d46c70d8059f20cab655720ec5792d18b71daad0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 13:11:25 +0000 Subject: [PATCH 0530/1392] Bump github/codeql-action from 2 to 3 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0d4d81efd..ae5241898 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -47,7 +47,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} setup-python-dependencies: false @@ -62,7 +62,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -75,6 +75,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:${{matrix.language}}" From b12a54a2fc7e988667f95145833a0dc8402b084d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Dec 2023 19:51:34 -0500 Subject: [PATCH 0531/1392] Use Path.touch to create files for rmtree tests It's not necessary to use `write_bytes(b"")`, because pathlib.Path has `touch()`. --- test/test_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 6616e1067..9408a91a0 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -46,7 +46,7 @@ def permission_error_tmpdir(tmp_path): """Fixture to test permissions errors in situations where they are not overcome.""" td = tmp_path / "testdir" td.mkdir() - (td / "x").write_bytes(b"") + (td / "x").touch() # Set up PermissionError on Windows, where we can't delete read-only files. (td / "x").chmod(stat.S_IRUSR) @@ -73,7 +73,7 @@ def test_deletes_nested_dir_with_files(self, tmp_path): td / "s" / "y", td / "s" / "z", ): - f.write_bytes(b"") + f.touch() try: rmtree(td) @@ -95,7 +95,7 @@ def test_deletes_dir_with_readonly_files(self, tmp_path): for d in td, td / "sub": d.mkdir() for f in td / "x", td / "sub" / "y": - f.write_bytes(b"") + f.touch() f.chmod(0) try: @@ -115,7 +115,7 @@ def test_avoids_changing_permissions_outside_tree(self, tmp_path): dir1 = tmp_path / "dir1" dir1.mkdir() - (dir1 / "file").write_bytes(b"") + (dir1 / "file").touch() (dir1 / "file").chmod(stat.S_IRUSR) old_mode = (dir1 / "file").stat().st_mode From 6a8ed70a6003c13800d908ab055038eb61ba4ce9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Dec 2023 20:00:56 -0500 Subject: [PATCH 0532/1392] Run test_env_vars_for_windows_tests only on Windows This skips the tests of how the HIDE_WINDOWS_KNOWN_ERRORS and HIDE_WINDOWS_FREEZE_ERRORS environment variables affect the same-named attributes of git.util, except when testing on Windows. These are parsed only to ever set a True value on Windows, but checking that this is the case is less important ever since git.util.rmtree was changed to not check HIDE_WINDOWS_KNOWN_ERRORS on other systems (and this is covered in other tests). Setting the variables to True on non-Windows systems would still have a bad effect on the tests themselves, some of which use them as skip or xfail conditions separate from the skipping logic in git.util.rmtree. However, this is effectively using them as part of the test suite (which they were initially meant for and which they may eventually go back to being, for #790), where they would not ordinarily have tests. The benefit and motivation for running these tests only on Windows is that the tests can be simplified, so that their parameter sets are no longer confusing. That change is also made here. --- test/test_util.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 9408a91a0..f3769088c 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -207,24 +207,28 @@ def _run_parse(name, value): ) return ast.literal_eval(output) + @pytest.mark.skipif( + os.name != "nt", + reason="These environment variables are only used on Windows.", + ) @pytest.mark.parametrize( "env_var_value, expected_truth_value", [ - (None, os.name == "nt"), # True on Windows when the environment variable is unset. + (None, True), # When the environment variable is unset. ("", False), (" ", False), ("0", False), - ("1", os.name == "nt"), + ("1", True), ("false", False), - ("true", os.name == "nt"), + ("true", True), ("False", False), - ("True", os.name == "nt"), + ("True", True), ("no", False), - ("yes", os.name == "nt"), + ("yes", True), ("NO", False), - ("YES", os.name == "nt"), + ("YES", True), (" no ", False), - (" yes ", os.name == "nt"), + (" yes ", True), ], ) @pytest.mark.parametrize( From 487a4fdb38efc82d30111e49f941a25cba4aa7a7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Dec 2023 22:39:11 -0500 Subject: [PATCH 0533/1392] Add a test for git.index.util.TemporaryFileSwap This is a general test for TemporaryFileSwap, but by being parametrized by the type of file_path, it reveals a regression introduced in 9e86053 (#1770). TemporaryFileSwap still works when file_path is a string, but is now broken when it is a Path. That worked before, and the type annotations document that it should be able to work. This is at least a bug because TemporaryFileSwap is public. (I am unsure whether, in practice, GitPython itself uses it in a way that sometimes passes a Path object as file_path. But code that uses GitPython may call it directly and pass Path.) --- test/test_index.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/test/test_index.py b/test/test_index.py index 2f97f0af8..c3f3b4fae 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -34,10 +34,11 @@ ) from git.index.fun import hook_path from git.index.typ import BaseIndexEntry, IndexEntry +from git.index.util import TemporaryFileSwap from git.objects import Blob -from test.lib import TestBase, fixture, fixture_path, with_rw_directory, with_rw_repo from git.util import Actor, hex_to_bin, rmtree from gitdb.base import IStream +from test.lib import TestBase, fixture, fixture_path, with_rw_directory, with_rw_repo HOOKS_SHEBANG = "#!/usr/bin/env sh\n" @@ -1087,3 +1088,25 @@ def test_index_add_pathlike(self, rw_repo): file.touch() rw_repo.index.add(file) + + +class TestIndexUtils: + @pytest.mark.parametrize("file_path_type", [str, Path]) + def test_temporary_file_swap(self, tmp_path, file_path_type): + file_path = tmp_path / "foo" + file_path.write_bytes(b"some data") + + with TemporaryFileSwap(file_path_type(file_path)) as ctx: + assert Path(ctx.file_path) == file_path + assert not file_path.exists() + + # Recreate it with new data, so we can observe that they're really separate. + file_path.write_bytes(b"other data") + + temp_file_path = Path(ctx.tmp_file_path) + assert temp_file_path.parent == file_path.parent + assert temp_file_path.name.startswith(file_path.name) + assert temp_file_path.read_bytes() == b"some data" + + assert not temp_file_path.exists() + assert file_path.read_bytes() == b"some data" # Not b"other data". From 1ddf953ea0d7625485ed02c90b03aae7f939ec46 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Dec 2023 22:58:17 -0500 Subject: [PATCH 0534/1392] Fix TemporaryFileSwap bug when file_path is a Path This fixes the regression introduced in 9e86053 (#1770) where the file_path argument to TemporaryFileSwap.__init__ could no longer be a Path object. The change also makes this truer to the code from before #1770, still without the race condition fixed there, in that str was called on file_path then as well. However, it is not clear that this is a good thing, because this is not an idiomatic use of mkstemp. The reason the `prefix` cannot be a Path is that it is expected to be a filename prefix, with leading directories given in the `dir` argument. --- git/index/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/index/util.py b/git/index/util.py index 1c3b1c4ad..03de80f71 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -41,7 +41,7 @@ class TemporaryFileSwap: def __init__(self, file_path: PathLike) -> None: self.file_path = file_path - fd, self.tmp_file_path = tempfile.mkstemp(prefix=self.file_path, dir="") + fd, self.tmp_file_path = tempfile.mkstemp(prefix=str(self.file_path), dir="") os.close(fd) with contextlib.suppress(OSError): # It may be that the source does not exist. os.replace(self.file_path, self.tmp_file_path) From b438c459d91243b685f540ccb1c124260e9d28b9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Dec 2023 23:29:14 -0500 Subject: [PATCH 0535/1392] Refactor TemporaryFileSwap.__init__ for clarity This changes the mkstemp call TemporaryFileSwap uses to atomically create (and thus reserve) the temporary file, so that it passes only a filename (i.e., basename) prefix as `prefix`, and passes all other path components (i.e., the directory) as `dir`. (If the original path has no directory, then `dir` is "" as before.) This makes the mkstemp call *slightly* more idiomatic. This also makes it clearer, as it is no longer using `prefix` for something that feels like it should be possible to pass as a Path object. Although mkstemp does not accept a Path as `prefix`, it does (as expected) accept one as `dir`. However, to keep the code simple, I'm passing str for both. The os.path.split function accepts both str and Path (since Python 3.6), and returns str objects, which are now used for the `dir` and `prefix` arguments to mkstemp. For unusual cases, this may technically not be a refactoring. For example, a file_path of "a/b//c" will be split into "a/b" and "c". If the automatically generated temporary file suffix is "xyz", then that results in a tmp_file_path of "a/b/cxyz" where "a/b//cxyz" would have been used before. The tmp_file_path attribute of a TemporaryFileSwap object is public (and used in filesystem calls). However, no guarantee has ever been given that the temporary file path have the original path as an exact string prefix. I believe the slightly weaker relationship I expressed in the recently introduced test_temporary_file_swap -- another file in the same directory, named with the original filename with more characters, consisting of equivalent path components in the same order -- has always been the intended one. Note that this slight possible variation does not apply to the file_path attribute. That attribute is always kept exactly as it was, both in its type and its value, and it always used unmodified in calls that access the filesystem. --- git/index/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/git/index/util.py b/git/index/util.py index 03de80f71..61039fe7c 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -41,7 +41,8 @@ class TemporaryFileSwap: def __init__(self, file_path: PathLike) -> None: self.file_path = file_path - fd, self.tmp_file_path = tempfile.mkstemp(prefix=str(self.file_path), dir="") + dirname, basename = osp.split(file_path) + fd, self.tmp_file_path = tempfile.mkstemp(prefix=basename, dir=dirname) os.close(fd) with contextlib.suppress(OSError): # It may be that the source does not exist. os.replace(self.file_path, self.tmp_file_path) From 4e91a6c7b521dc7d0331dbb5455e9c424d26d655 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Dec 2023 23:56:55 -0500 Subject: [PATCH 0536/1392] Tweak formatting for `@pytest.mark.parametrize` This removes the "fmt: off" / "fmt: on" directives around the `@pytest.mark.parametrize` decoration on test_blob_filter, and reformats it with black, for consistency with other such decorations. The style used there, *if* it could be specified as a rule and thus used without "fmt:" directives, may be nicer than how black formats multi-line mark decorations. However, since that decoration was written, there have been a number of other such decorations, which have been in black style. This also removes the only (or only remaining?) "fmt:" directive in the codebase. As such, it should possibly have been done in #1760. --- test/test_blob_filter.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/test_blob_filter.py b/test/test_blob_filter.py index a91f211bf..ddd83079a 100644 --- a/test/test_blob_filter.py +++ b/test/test_blob_filter.py @@ -14,14 +14,15 @@ from git.types import PathLike -# fmt: off -@pytest.mark.parametrize('paths, path, expected_result', [ - ((Path("foo"),), Path("foo"), True), - ((Path("foo"),), Path("foo/bar"), True), - ((Path("foo/bar"),), Path("foo"), False), - ((Path("foo"), Path("bar")), Path("foo"), True), -]) -# fmt: on +@pytest.mark.parametrize( + "paths, path, expected_result", + [ + ((Path("foo"),), Path("foo"), True), + ((Path("foo"),), Path("foo/bar"), True), + ((Path("foo/bar"),), Path("foo"), False), + ((Path("foo"), Path("bar")), Path("foo"), True), + ], +) def test_blob_filter(paths: Sequence[PathLike], path: PathLike, expected_result: bool) -> None: """Test the blob filter.""" blob_filter = BlobFilter(paths) From f664a0b85dd6be4bb8fd16d7283fd1be0f81c8c8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 21 Dec 2023 00:58:33 -0500 Subject: [PATCH 0537/1392] Add xfail marks to hook tests for WinBashStatus.Absent Precise xfail marks were added to commit hook tests in #1745, but in a few tests I didn't get it right for WinBashStatus.Absent. That is, on a Windows system that has no bash.exe at all: - More of the tests are unable to pass than have xfail marks. - One of the tests, test_commit_msg_hook_success, which does have an xfail mark for that, wrongly combines it with the xfail mark for WinBashStatus.Wsl. That test is the only one where the WSL bash.exe, even when a working WSL distribution is installed, is unable to pass. But using a single mark there is wrong, in part because the "reason" is not correct for both, but even more so because the exceptions they raise are different: AssertionError is raised when the WSL bash.exe is used in that test, but when bash.exe is altogether absent, HookExecutionError is raised. This fixes that by adding xfail marks for WinBashStatus.Absent where missing, and splitting test_commit_msg_hook_success's xfail mark that unsuccessfully tried to cover two conditions into two separate marks, each of which gives a correct reason and exception. This commit also rewords the xfail reason given for WslNoDistro, which was somewhat unclear and potentially ambiguous, to make it clearer. --- test/test_index.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index 2f97f0af8..3f0990ecc 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -991,9 +991,14 @@ class Mocked: rel = index._to_relative_path(path) self.assertEqual(rel, os.path.relpath(path, root)) + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.Absent, + reason="Can't run a hook on Windows without bash.exe.", + rasies=HookExecutionError, + ) @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, - reason="Currently uses the bash.exe for WSL even with no WSL distro installed", + reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", raises=HookExecutionError, ) @with_rw_repo("HEAD", bare=True) @@ -1004,7 +1009,7 @@ def test_pre_commit_hook_success(self, rw_repo): @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, - reason="Currently uses the bash.exe for WSL even with no WSL distro installed", + reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", raises=AssertionError, ) @with_rw_repo("HEAD", bare=True) @@ -1030,13 +1035,18 @@ def test_pre_commit_hook_fail(self, rw_repo): raise AssertionError("Should have caught a HookExecutionError") @pytest.mark.xfail( - type(_win_bash_status) in {WinBashStatus.Absent, WinBashStatus.Wsl}, + type(_win_bash_status) is WinBashStatus.Absent, + reason="Can't run a hook on Windows without bash.exe.", + rasies=HookExecutionError, + ) + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.Wsl, reason="Specifically seems to fail on WSL bash (in spite of #1399)", raises=AssertionError, ) @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, - reason="Currently uses the bash.exe for WSL even with no WSL distro installed", + reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", raises=HookExecutionError, ) @with_rw_repo("HEAD", bare=True) @@ -1054,7 +1064,7 @@ def test_commit_msg_hook_success(self, rw_repo): @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, - reason="Currently uses the bash.exe for WSL even with no WSL distro installed", + reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", raises=AssertionError, ) @with_rw_repo("HEAD", bare=True) From e1486474a2381763e11932b3bb15e08f6e9faf53 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 21 Dec 2023 01:31:54 -0500 Subject: [PATCH 0538/1392] Add a direct test of run_commit_hook This has three benefits: - run_commit_hook is public, being listed in git.index.fun.__all__, so it makes sense for it to have its own test. - When investigating (future, or current xfail-covered) failure of functions that use run_commit_hook, it will be useful to compare the results of other tests that already do exist to that of a direct test of run_commit_hook. - When changing which bash.exe run_commit_hook selects to use on Windows (including to ameliorate the limitation covered by the WinBashStatus.WslNoDistro xfail marks, or to attempt other possible changes suggested in #1745), or even just to investigate the possibility of doing so, it will make sense to add tests like this test but for more specific conditions or edge cases. Having this typical-case test to compare to should be helpful both for writing such tests and for efficiently verifying that the conditions they test are really the triggers for any failures. --- test/test_index.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test/test_index.py b/test/test_index.py index 3f0990ecc..afceda131 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -32,7 +32,7 @@ InvalidGitRepositoryError, UnmergedEntriesError, ) -from git.index.fun import hook_path +from git.index.fun import hook_path, run_commit_hook from git.index.typ import BaseIndexEntry, IndexEntry from git.objects import Blob from test.lib import TestBase, fixture, fixture_path, with_rw_directory, with_rw_repo @@ -991,6 +991,24 @@ class Mocked: rel = index._to_relative_path(path) self.assertEqual(rel, os.path.relpath(path, root)) + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.Absent, + reason="Can't run a hook on Windows without bash.exe.", + rasies=HookExecutionError, + ) + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.WslNoDistro, + reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", + raises=HookExecutionError, + ) + @with_rw_repo("HEAD", bare=True) + def test_run_commit_hook(self, rw_repo): + index = rw_repo.index + _make_hook(index.repo.git_dir, "fake-hook", "echo 'ran fake hook' >output.txt") + run_commit_hook("fake-hook", index) + output = Path(rw_repo.git_dir, "output.txt").read_text(encoding="utf-8") + self.assertEqual(output, "ran fake hook\n") + @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Absent, reason="Can't run a hook on Windows without bash.exe.", From 6e4cee4fa708465cf714e36a9dc8b9b6b94acc0c Mon Sep 17 00:00:00 2001 From: Stefan Gmeiner Date: Thu, 21 Dec 2023 21:33:37 +0100 Subject: [PATCH 0539/1392] Fix Items of type PathLike --- git/index/base.py | 2 +- test/test_index.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 112ad3feb..cffb213a9 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -940,7 +940,7 @@ def _items_to_rela_paths( for item in items: if isinstance(item, (BaseIndexEntry, (Blob, Submodule))): paths.append(self._to_relative_path(item.path)) - elif isinstance(item, str): + elif isinstance(item, (str, os.PathLike)): paths.append(self._to_relative_path(item)) else: raise TypeError("Invalid item type: %r" % item) diff --git a/test/test_index.py b/test/test_index.py index 6b746b8b4..50d941e83 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -558,14 +558,16 @@ def test_index_mutation(self, rw_repo): def mixed_iterator(): count = 0 for entry in index.entries.values(): - type_id = count % 4 - if type_id == 0: # path + type_id = count % 5 + if type_id == 0: # path (str) yield entry.path - elif type_id == 1: # blob + elif type_id == 1: # path (PathLike) + yield Path(entry.path) + elif type_id == 2: # blob yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) - elif type_id == 2: # BaseIndexEntry + elif type_id == 3: # BaseIndexEntry yield BaseIndexEntry(entry[:4]) - elif type_id == 3: # IndexEntry + elif type_id == 4: # IndexEntry yield entry else: raise AssertionError("Invalid Type") From 53e73830f64e13a88b0e246fb825944a6fe2848e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Dec 2023 00:57:14 -0500 Subject: [PATCH 0540/1392] Remove explicit PushInfo/FetchInfo inheritance from object I had missed these in f78587f (#1725). --- git/remote.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/remote.py b/git/remote.py index 4055dba2e..98a421b3a 100644 --- a/git/remote.py +++ b/git/remote.py @@ -130,7 +130,7 @@ def to_progress_instance( return progress -class PushInfo(IterableObj, object): +class PushInfo(IterableObj): """ Carries information about the result of a push operation of a single head:: @@ -300,7 +300,7 @@ def raise_if_error(self) -> None: raise self.error -class FetchInfo(IterableObj, object): +class FetchInfo(IterableObj): """ Carries information about the results of a fetch operation of a single head:: From 96fc3547cf62c5ade1477c24566c8b34254a1507 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Dec 2023 01:55:30 -0500 Subject: [PATCH 0541/1392] Add tests for current Submodule.iter_items behavior Where the behavior is intended. In the case of an invalid hash (or IOError, which in Python 2 was a subclass of OSError but now is just another name for it), the behavior of just yielding no items may be unintuitive, since on most other errors an exception is raised. However, examining the code reveals this behavior is clearly intentional. Furthrmore, it may be reasonable for applications to rely on it, and it may be convenient in some situations. For backward compatibility, it probably can't be changed significantly. This adds tests that show both an error that does raise an error-representing exception -- a well-formed hash not present in the repository raising ValueError with a suitable message -- and an error that silently causes the iterator to yield zero items. --- test/test_submodule.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test_submodule.py b/test/test_submodule.py index 4dc89f98f..5226d7a6e 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -688,6 +688,17 @@ def test_root_module(self, rwrepo): # gitdb: has either 1 or 2 submodules depending on the version. assert len(nsm.children()) >= 1 and nsmc.module_exists() + def test_iter_items_from_nonexistent_hash(self): + it = Submodule.iter_items(self.rorepo, "b4ecbfaa90c8be6ed6d9fb4e57cc824663ae15b4") + with self.assertRaisesRegex(ValueError, r"\bcould not be resolved\b"): + next(it) + + def test_iter_items_from_invalid_hash(self): + """Check legacy behavaior on BadName (also applies to IOError, i.e. OSError).""" + it = Submodule.iter_items(self.rorepo, "xyz") + with self.assertRaises(StopIteration): + next(it) + @with_rw_repo(k_no_subm_tag, bare=False) def test_first_submodule(self, rwrepo): assert len(list(rwrepo.iter_submodules())) == 0 From f5dc1c4713dfb937d45d10a595ac879d6e76481c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Dec 2023 02:16:28 -0500 Subject: [PATCH 0542/1392] Expand "invalid hash" test to assert normal StopIteration Returning an explicit value from a generator function causes that value to be bound to the `value` attribute of the StopIteration exception. This is available as the result of "yield from" when it is used as an expression; or by explicitly catching StopIteration, binding the StopIteration exception to a variable, and accessing the attribute. This feature of generators is rarely used. The `return iter([])` statement in Submodule.iter_items uses this feature, causing the resulting StopIteration exception object to have a `value` attribute that refers to a separate second iterator that also yields no values (#1779). From context, this behavior is clearly not the goal; a bare return statement should be used here (which has the same effect except for the `value` attribute of the StopIteration exception). The code had used a bare return prior to 82b131c (#1282), when `return` was changed to `return iter([])`. That was part of a change that added numerous type annotations. It looks like it was either a mistake, or possibly an attempt to work around an old bug in a static type checker. This commit extends the test_iter_items_from_invalid_hash test to assert that the `value` attribute of the StopIteration is its usual default value of None. This commit only extends the test; it does not fix the bug. --- test/test_submodule.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 5226d7a6e..993f6b57e 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -696,8 +696,9 @@ def test_iter_items_from_nonexistent_hash(self): def test_iter_items_from_invalid_hash(self): """Check legacy behavaior on BadName (also applies to IOError, i.e. OSError).""" it = Submodule.iter_items(self.rorepo, "xyz") - with self.assertRaises(StopIteration): + with self.assertRaises(StopIteration) as ctx: next(it) + self.assertIsNone(ctx.exception.value) @with_rw_repo(k_no_subm_tag, bare=False) def test_first_submodule(self, rwrepo): From c3c008c4971f9d4189dc08e88334a207ce14298c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Dec 2023 02:44:56 -0500 Subject: [PATCH 0543/1392] In Submodule.iter_items, don't attach second empty iterator This fixes the minor bug where a separate empty iterator was bound to the StopIteration exception raised as a result of returning from the generator function (#1779). This change does not cause what exceptions are raised from GitPython code in any situations, nor how many items any iterators yield. --- git/objects/submodule/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 651d9535c..49dfedf9a 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1401,7 +1401,7 @@ def iter_items( pc = repo.commit(parent_commit) # Parent commit instance parser = cls._config_parser(repo, pc, read_only=True) except (IOError, BadName): - return iter([]) + return # END handle empty iterator for sms in parser.sections(): From dfee31f2100d7f4a653c69c4a5a505607fe328e1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Dec 2023 03:48:57 -0500 Subject: [PATCH 0544/1392] Improve self-documentation of IterableObj and related classes - Fill in the missing part of the explanation of why to favor iter_items over list_items in IterableObj and Iterable (#1775). - Move the explanation of how subclasses must treat arguments from the list_items methods to the iter_items methods, because the iter_items methdos are the ones that are abstract and must be overridden by a well-behaved subclass, and also because, since the iter_items methods are preferred for use, they should be the place where less duplicated shared documentation resides. - Subtantially reword the existing documentation for clarity, especially regarding the signifance of extra args and kwargs. - Put the iter_items method before (i.e. above) the list_items method (in each of the IterableObj and Iterable classes), because that method is the one that should be used more often, and because it is also now the one with the more detailed docstring. - Remove and old comment on a return type that said exactly the exact same thing as the annotation. - In Iterable, note deprecation more consistently (and thus in more places). - Rewrite the IterableClassWatcher class docstring to explain exactly what that metaclass achieves. --- git/util.py | 84 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/git/util.py b/git/util.py index 0a5da7d71..5acc001f7 100644 --- a/git/util.py +++ b/git/util.py @@ -1183,7 +1183,8 @@ def __delitem__(self, index: Union[SupportsIndex, int, slice, str]) -> None: class IterableClassWatcher(type): - """Metaclass that watches.""" + """Metaclass that issues :class:`DeprecationWarning` when :class:`git.util.Iterable` + is subclassed.""" def __init__(cls, name: str, bases: Tuple, clsdict: Dict) -> None: for base in bases: @@ -1199,23 +1200,42 @@ def __init__(cls, name: str, bases: Tuple, clsdict: Dict) -> None: class Iterable(metaclass=IterableClassWatcher): - """Defines an interface for iterable items, so there is a uniform way to retrieve - and iterate items within the git repository.""" + """Deprecated, use :class:`IterableObj` instead. + + Defines an interface for iterable items, so there is a uniform way to retrieve + and iterate items within the git repository. + """ __slots__ = () _id_attribute_ = "attribute that most suitably identifies your instance" @classmethod - def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: + def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: + # return typed to be compatible with subtypes e.g. Remote + """Deprecated, use :class:`IterableObj` instead. + + Find (all) items of this type. + + Subclasses can specify ``args`` and ``kwargs`` differently, and may use them for + filtering. However, when the method is called with no additional positional or + keyword arguments, subclasses are obliged to to yield all items. + + :return: Iterator yielding Items """ - Deprecated, use IterableObj instead. + raise NotImplementedError("To be implemented by Subclass") + + @classmethod + def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: + """Deprecated, use :class:`IterableObj` instead. + + Find (all) items of this type and collect them into a list. - Find all items of this type - subclasses can specify args and kwargs differently. - If no args are given, subclasses are obliged to return all items if no additional - arguments arg given. + For more information about the arguments, see :meth:`list_items`. - :note: Favor the iter_items method as it will + :note: Favor the :meth:`iter_items` method as it will avoid eagerly collecting + all items. When there are many items, that can slow performance and increase + memory usage. :return: list(Item,...) list of item instances """ @@ -1223,15 +1243,6 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: out_list.extend(cls.iter_items(repo, *args, **kwargs)) return out_list - @classmethod - def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: - # return typed to be compatible with subtypes e.g. Remote - """For more information about the arguments, see list_items. - - :return: Iterator yielding Items - """ - raise NotImplementedError("To be implemented by Subclass") - @runtime_checkable class IterableObj(Protocol): @@ -1246,13 +1257,30 @@ class IterableObj(Protocol): _id_attribute_: str @classmethod - def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> IterableList[T_IterableObj]: + @abstractmethod + def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator[T_IterableObj]: + # Return-typed to be compatible with subtypes e.g. Remote. + """Find (all) items of this type. + + Subclasses can specify ``args`` and ``kwargs`` differently, and may use them for + filtering. However, when the method is called with no additional positional or + keyword arguments, subclasses are obliged to to yield all items. + + For more information about the arguments, see list_items. + + :return: Iterator yielding Items """ - Find all items of this type - subclasses can specify args and kwargs differently. - If no args are given, subclasses are obliged to return all items if no additional - arguments arg given. + raise NotImplementedError("To be implemented by Subclass") + + @classmethod + def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> IterableList[T_IterableObj]: + """Find (all) items of this type and collect them into a list. + + For more information about the arguments, see :meth:`list_items`. - :note: Favor the iter_items method as it will + :note: Favor the :meth:`iter_items` method as it will avoid eagerly collecting + all items. When there are many items, that can slow performance and increase + memory usage. :return: list(Item,...) list of item instances """ @@ -1260,16 +1288,6 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> IterableList[T_I out_list.extend(cls.iter_items(repo, *args, **kwargs)) return out_list - @classmethod - @abstractmethod - def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator[T_IterableObj]: # Iterator[T_IterableObj]: - # Return-typed to be compatible with subtypes e.g. Remote. - """For more information about the arguments, see list_items. - - :return: Iterator yielding Items - """ - raise NotImplementedError("To be implemented by Subclass") - # } END classes From 94a85d1159e6374e901df4e3bd284cf55b623e66 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Dec 2023 14:31:28 -0500 Subject: [PATCH 0545/1392] Convert constant and attribute comments to docstrings These are "non-reified docstrings" as described in #1734. They are not made part of the data model, but most editors will display them, including on symbols present in code of other projects that use the GitPython library. A number of these have already been added, but this adds what will probably be most of the remaining ones. For the most part, this doesn't create documentation where there wasn't any, but instead converts existing comments to "docstrings." In a handful of cases, they are expanded, reworded, or a docstring added. This also fixes some small style inconsistencies that were missed in #1725, and moves a comment that had become inadvertently displaced due to autoformatting to the item it was meant for. The major omission here is HIDE_WINDOWS_KNOWN_ERRORS and HIDE_WINDOWS_FREEZE_ERRORS. This doesn't convert the comments above them to "docstrings," for a few reasons. They are not specific to either of the symbols, they are oriented toward considerations that are not really relevant except when developing GitPython itself, and they are out of date. Also, because HIDE_WINDOWS_KNOWN_ERRORS is listed in __all__, increasing the level of documentation for it might be taken as a committment to preserve some aspect of its current behavior, which could interfere with progress on #790. So I've kept those comments as comments, and unchanged, for now. --- git/cmd.py | 25 ++++++++++++++++--------- git/config.py | 20 ++++++++++++-------- git/diff.py | 2 +- git/index/base.py | 6 ++++-- git/index/fun.py | 4 +++- git/objects/submodule/base.py | 11 ++++++----- git/repo/base.py | 31 ++++++++++++++++++++----------- git/util.py | 4 ++-- 8 files changed, 64 insertions(+), 39 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 27148d3d6..9518c2c8c 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -273,23 +273,30 @@ def __setstate__(self, d: Dict[str, Any]) -> None: # CONFIGURATION - git_exec_name = "git" # Default that should work on Linux and Windows. + git_exec_name = "git" + """Default git command that should work on Linux, Windows, and other systems.""" - # Enables debugging of GitPython's git commands. GIT_PYTHON_TRACE = os.environ.get("GIT_PYTHON_TRACE", False) + """Enables debugging of GitPython's git commands.""" - # If True, a shell will be used when executing git commands. - # This should only be desirable on Windows, see https://github.com/gitpython-developers/GitPython/pull/126 - # and check `git/test_repo.py:TestRepo.test_untracked_files()` TC for an example where it is required. - # Override this value using `Git.USE_SHELL = True`. USE_SHELL = False + """If True, a shell will be used when executing git commands. + + This should only be desirable on Windows, see https://github.com/gitpython-developers/GitPython/pull/126 + and check `git/test_repo.py:TestRepo.test_untracked_files()` TC for an example where it is required. + + Override this value using ``Git.USE_SHELL = True``. + """ - # Provide the full path to the git executable. Otherwise it assumes git is in the path. _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" _refresh_env_var = "GIT_PYTHON_REFRESH" + GIT_PYTHON_GIT_EXECUTABLE = None - # Note that the git executable is actually found during the refresh step in - # the top level __init__. + """Provide the full path to the git executable. Otherwise it assumes git is in the path. + + Note that the git executable is actually found during the refresh step in + the top level ``__init__``. + """ @classmethod def refresh(cls, path: Union[None, PathLike] = None) -> bool: diff --git a/git/config.py b/git/config.py index 5708a7385..2730ddaf3 100644 --- a/git/config.py +++ b/git/config.py @@ -65,12 +65,14 @@ log.addHandler(logging.NullHandler()) -# The configuration level of a configuration file. CONFIG_LEVELS: ConfigLevels_Tup = ("system", "user", "global", "repository") +"""The configuration level of a configuration file.""" -# Section pattern to detect conditional includes. -# https://git-scm.com/docs/git-config#_conditional_includes CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeIf )\"(gitdir|gitdir/i|onbranch):(.+)\"") +"""Section pattern to detect conditional includes. + +See: https://git-scm.com/docs/git-config#_conditional_includes +""" class MetaParserBuilder(abc.ABCMeta): # noqa: B024 @@ -283,12 +285,14 @@ class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): """ # { Configuration - # The lock type determines the type of lock to use in new configuration readers. - # They must be compatible to the LockFile interface. - # A suitable alternative would be the BlockingLockFile t_lock = LockFile - re_comment = re.compile(r"^\s*[#;]") + """The lock type determines the type of lock to use in new configuration readers. + They must be compatible to the LockFile interface. + A suitable alternative would be the :class:`~git.util.BlockingLockFile`. + """ + + re_comment = re.compile(r"^\s*[#;]") # } END configuration optvalueonly_source = r"\s*(?P

``. - One of these helpers (``ext::``) can be used to invoke any arbitrary command. + Apart from the usual protocols (http, git, ssh), Git allows "remote helpers" + that have the form ``::
``. One of these helpers (``ext::``) + can be used to invoke any arbitrary command. See: @@ -592,11 +611,11 @@ def check_unsafe_protocols(cls, url: str) -> None: def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> None: """Check for unsafe options. - Some options that are passed to `git ` can be used to execute - arbitrary commands, this are blocked by default. + Some options that are passed to ``git `` can be used to execute + arbitrary commands. These are blocked by default. """ - # Options can be of the form `foo` or `--foo bar` `--foo=bar`, - # so we need to check if they start with "--foo" or if they are equal to "foo". + # Options can be of the form `foo`, `--foo bar`, or `--foo=bar`, so we need to + # check if they start with "--foo" or if they are equal to "foo". bare_unsafe_options = [option.lstrip("-") for option in unsafe_options] for option in options: for unsafe_option, bare_option in zip(unsafe_options, bare_unsafe_options): @@ -673,9 +692,15 @@ def __getattr__(self, attr: str) -> Any: def wait(self, stderr: Union[None, str, bytes] = b"") -> int: """Wait for the process and return its status code. - :param stderr: Previously read value of stderr, in case stderr is already closed. - :warn: May deadlock if output or error pipes are used and not handled separately. - :raise GitCommandError: If the return status is not 0. + :param stderr: + Previously read value of stderr, in case stderr is already closed. + + :warn: + May deadlock if output or error pipes are used and not handled + separately. + + :raise GitCommandError: + If the return status is not 0. """ if stderr is None: stderr_b = b"" @@ -725,8 +750,8 @@ def __init__(self, size: int, stream: IO[bytes]) -> None: self._size = size self._nbr = 0 # Number of bytes read. - # Special case: If the object is empty, has null bytes, get the - # final newline right away. + # Special case: If the object is empty, has null bytes, get the final + # newline right away. if size == 0: stream.read(1) # END handle empty streams @@ -745,7 +770,8 @@ def read(self, size: int = -1) -> bytes: data = self._stream.read(size) self._nbr += len(data) - # Check for depletion, read our final byte to make the stream usable by others. + # Check for depletion, read our final byte to make the stream usable by + # others. if self._size - self._nbr == 0: self._stream.read(1) # final newline # END finish reading @@ -820,7 +846,7 @@ def __init__(self, working_dir: Union[None, PathLike] = None): :param working_dir: Git directory we should work in. If None, we always work in the current directory as returned by :func:`os.getcwd`. - It is meant to be the working tree directory if available, or the + This is meant to be the working tree directory if available, or the ``.git`` directory in case of bare repositories. """ super().__init__() @@ -844,8 +870,8 @@ def __getattr__(self, name: str) -> Any: an object. :return: - Callable object that will execute call :meth:`_call_process` with - your arguments. + Callable object that will execute call :meth:`_call_process` with your + arguments. """ if name.startswith("_"): return super().__getattribute__(name) @@ -857,8 +883,8 @@ def set_persistent_git_options(self, **kwargs: Any) -> None: :param kwargs: A dict of keyword arguments. - These arguments are passed as in :meth:`_call_process`, but will be - passed to the git command rather than the subcommand. + These arguments are passed as in :meth:`_call_process`, but will be passed + to the git command rather than the subcommand. """ self._persistent_git_options = self.transform_kwargs(split_single_char_options=True, **kwargs) @@ -872,7 +898,7 @@ def working_dir(self) -> Union[None, PathLike]: def version_info(self) -> Tuple[int, ...]: """ :return: tuple with integers representing the major, minor and additional - version numbers as parsed from git version. Up to four fields are used. + version numbers as parsed from ``git version``. Up to four fields are used. This value is generated on demand and is cached. """ @@ -1021,8 +1047,8 @@ def execute( If True, default True, we open stdout on the created process. :param universal_newlines: - if True, pipes will be opened as text, and lines are split at - all known line endings. + If True, pipes will be opened as text, and lines are split at all known line + endings. :param shell: Whether to invoke commands through a shell (see `Popen(..., shell=True)`). @@ -1057,6 +1083,7 @@ def execute( * tuple(int(status), str(stdout), str(stderr)) if extended_output = True If output_stream is True, the stdout value will be your output stream: + * output_stream if extended_output = False * tuple(int(status), output_stream, str(stderr)) if extended_output = True @@ -1254,14 +1281,16 @@ def update_environment(self, **kwargs: Any) -> Dict[str, Union[str, None]]: values in a format that can be passed back into this function to revert the changes. - ``Examples``:: + Examples:: old_env = self.update_environment(PWD='/tmp') self.update_environment(**old_env) - :param kwargs: Environment variables to use for git processes + :param kwargs: + Environment variables to use for git processes - :return: Dict that maps environment variables to their old values + :return: + Dict that maps environment variables to their old values """ old_env = {} for key, value in kwargs.items(): @@ -1280,12 +1309,13 @@ def custom_environment(self, **kwargs: Any) -> Iterator[None]: """A context manager around the above :meth:`update_environment` method to restore the environment back to its previous state after operation. - ``Examples``:: + Examples:: with self.custom_environment(GIT_SSH='/bin/ssh_wrapper'): repo.remotes.origin.fetch() - :param kwargs: See :meth:`update_environment` + :param kwargs: + See :meth:`update_environment`. """ old_env = self.update_environment(**kwargs) try: @@ -1310,7 +1340,7 @@ def transform_kwarg(self, name: str, value: Any, split_single_char_options: bool return [] def transform_kwargs(self, split_single_char_options: bool = True, **kwargs: Any) -> List[str]: - """Transform Python style kwargs into git command line options.""" + """Transform Python-style kwargs into git command line options.""" args = [] for k, v in kwargs.items(): if isinstance(v, (list, tuple)): @@ -1339,7 +1369,8 @@ def __call__(self, **kwargs: Any) -> "Git": These arguments are passed as in :meth:`_call_process`, but will be passed to the git command rather than the subcommand. - ``Examples``:: + Examples:: + git(work_tree='/tmp').difftool() """ self._git_options = self.transform_kwargs(split_single_char_options=True, **kwargs) @@ -1369,23 +1400,25 @@ def _call_process( def _call_process( self, method: str, *args: Any, **kwargs: Any ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], "Git.AutoInterrupt"]: - """Run the given git command with the specified arguments and return - the result as a string. + """Run the given git command with the specified arguments and return the result + as a string. :param method: - The command. Contained ``_`` characters will be converted to dashes, - such as in ``ls_files`` to call ``ls-files``. + The command. Contained ``_`` characters will be converted to hyphens, such + as in ``ls_files`` to call ``ls-files``. :param args: The list of arguments. If None is included, it will be pruned. - This allows your commands to call git more conveniently as None - is realized as non-existent. + This allows your commands to call git more conveniently, as None is realized + as non-existent. :param kwargs: Contains key-values for the following: + - The :meth:`execute()` kwds, as listed in :var:`execute_kwargs`. - "Command options" to be converted by :meth:`transform_kwargs`. - The `'insert_kwargs_after'` key which its value must match one of ``*args``. + It also contains any command options, to be appended after the matched arg. Examples:: @@ -1396,7 +1429,8 @@ def _call_process( git rev-list max-count 10 --header master - :return: Same as :meth:`execute`. + :return: + Same as :meth:`execute`. If no args are given, used :meth:`execute`'s default (especially ``as_process = False``, ``stdout_as_string = True``) and return str. """ @@ -1447,8 +1481,8 @@ def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: :return: (hex_sha, type_string, size_as_int) - :raise ValueError: If the header contains indication for an error due to - incorrect input sha + :raise ValueError: + If the header contains indication for an error due to incorrect input sha """ tokens = header_line.split() if len(tokens) != 3: @@ -1504,22 +1538,27 @@ def __get_object_header(self, cmd: "Git.AutoInterrupt", ref: AnyStr) -> Tuple[st raise ValueError("cmd stdin was empty") def get_object_header(self, ref: str) -> Tuple[str, str, int]: - """Use this method to quickly examine the type and size of the object behind - the given ref. + """Use this method to quickly examine the type and size of the object behind the + given ref. - :note: The method will only suffer from the costs of command invocation - once and reuses the command in subsequent calls. + :note: + The method will only suffer from the costs of command invocation once and + reuses the command in subsequent calls. - :return: (hexsha, type_string, size_as_int) + :return: + (hexsha, type_string, size_as_int) """ cmd = self._get_persistent_cmd("cat_file_header", "cat_file", batch_check=True) return self.__get_object_header(cmd, ref) def get_object_data(self, ref: str) -> Tuple[str, str, int, bytes]: - """As get_object_header, but returns object data as well. + """Similar to :meth:`get_object_header`, but returns object data as well. + + :return: + (hexsha, type_string, size_as_int, data_string) - :return: (hexsha, type_string, size_as_int, data_string) - :note: Not threadsafe. + :note: + Not threadsafe. """ hexsha, typename, size, stream = self.stream_object_data(ref) data = stream.read(size) @@ -1527,10 +1566,14 @@ def get_object_data(self, ref: str) -> Tuple[str, str, int, bytes]: return (hexsha, typename, size, data) def stream_object_data(self, ref: str) -> Tuple[str, str, int, "Git.CatFileContentStream"]: - """As get_object_header, but returns the data as a stream. + """Similar to :meth:`get_object_header`, but returns the data as a stream. - :return: (hexsha, type_string, size_as_int, stream) - :note: This method is not threadsafe, you need one independent Command instance per thread to be safe! + :return: + (hexsha, type_string, size_as_int, stream) + + :note: + This method is not threadsafe. You need one independent Command instance per + thread to be safe! """ cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True) hexsha, typename, size = self.__get_object_header(cmd, ref) @@ -1542,7 +1585,8 @@ def clear_cache(self) -> "Git": Currently persistent commands will be interrupted. - :return: self + :return: + self """ for cmd in (self.cat_file_all, self.cat_file_header): if cmd: From 8bb882ef914ac7e39cdc93547d012f8503730849 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 24 Feb 2024 15:07:33 -0500 Subject: [PATCH 0665/1392] Fix concurrency note for stream_object_data --- git/cmd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 1206641dc..eb6c235c7 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1572,8 +1572,8 @@ def stream_object_data(self, ref: str) -> Tuple[str, str, int, "Git.CatFileConte (hexsha, type_string, size_as_int, stream) :note: - This method is not threadsafe. You need one independent Command instance per - thread to be safe! + This method is not threadsafe. You need one independent :class:`Git` + instance per thread to be safe! """ cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True) hexsha, typename, size = self.__get_object_header(cmd, ref) From ba878ef09228176c17112f90434c685f5535a98a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 24 Feb 2024 15:42:38 -0500 Subject: [PATCH 0666/1392] Reword partial_to_complete_sha_hex note The git.db.partial_to_complete_sha_hex docstring refers to "AmbiguousObjects", suggesting the existence of an AmbiguousObject type (or an AmbiguousObjects type). Since there appears to be no such type in GitPython (or in gitdb), this seems to have been written in reference to the condition expressed by the AmbiguousObjectName exception, which the note says is not currently able to be raised (such that BadObject is raised instead) in situations where it would apply. Since the connection to that exeception is already clear from context, this commit changes the wording to "ambiguous objects" to avoid being misread as a reference to a Python class of ambiguous objects. --- git/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/db.py b/git/db.py index 03b631084..1531d663c 100644 --- a/git/db.py +++ b/git/db.py @@ -59,7 +59,7 @@ def partial_to_complete_sha_hex(self, partial_hexsha: str) -> bytes: :raise BadObject: :note: Currently we only raise :class:`BadObject` as git does not communicate - AmbiguousObjects separately. + ambiguous objects separately. """ try: hexsha, _typename, _size = self._git.get_object_header(partial_hexsha) From 39587475ff47ce3a7986f0a6f7cd29cbcdcf01bb Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 24 Feb 2024 16:39:33 -0500 Subject: [PATCH 0667/1392] Update CommandError._msg documentation This converts it from a specially formatted comment to a docstring (#1734), rewords for clarity, and removes the mention of unicode, which appears to have been referring to the data type. (GitPython no longer supports Python 2, and unicode is not a separate type from str in Python 3, where instead str and bytes are different types.) --- git/exc.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/git/exc.py b/git/exc.py index 85113bc44..40fb8eea0 100644 --- a/git/exc.py +++ b/git/exc.py @@ -87,10 +87,13 @@ class CommandError(GitError): A non-empty list of argv comprising the command-line. """ - #: A unicode print-format with 2 `%s for `` and the rest, - #: e.g. - #: "'%s' failed%s" _msg = "Cmd('%s') failed%s" + """Format string with 2 ``%s`` for ```` and the rest. + + For example: ``"'%s' failed%s"`` + + Subclasses may override this attribute, provided it is still in this form. + """ def __init__( self, From f56e1ac5c5a548d5f74b355f34643053f9b0eb6c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 24 Feb 2024 17:00:02 -0500 Subject: [PATCH 0668/1392] Tweak code formatting in Remote._set_cache_ For slightly easier reading. --- git/remote.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/git/remote.py b/git/remote.py index 3ccac59de..5ae361097 100644 --- a/git/remote.py +++ b/git/remote.py @@ -575,7 +575,10 @@ def _set_cache_(self, attr: str) -> None: if attr == "_config_reader": # NOTE: This is cached as __getattr__ is overridden to return remote config # values implicitly, such as in print(r.pushurl). - self._config_reader = SectionConstraint(self.repo.config_reader("repository"), self._config_section_name()) + self._config_reader = SectionConstraint( + self.repo.config_reader("repository"), + self._config_section_name(), + ) else: super()._set_cache_(attr) From fa471fe7b58085e9676c23a74443460a5c1d6ed6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 24 Feb 2024 17:42:52 -0500 Subject: [PATCH 0669/1392] Fix up Remote.push docstring - Replace the goo.gl web shortlink with a Sphinx reference that displays and also (in built documentation) becomes a hyperlink to the documentation on the method being referred to. - Refer to a related class (which is in another module) as being in the module where it is defined, rather than in the top-level git module (where it also can be accessed because it is imported there, which is reasonable to do, but less clear documentation). - Tweak punctuation so the effect of passing None is a bit clearer. --- git/remote.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/git/remote.py b/git/remote.py index 5ae361097..df809a9ac 100644 --- a/git/remote.py +++ b/git/remote.py @@ -1075,13 +1075,14 @@ def push( :param progress: Can take one of many value types: - * None to discard progress information. + * None, to discard progress information. * A function (callable) that is called with the progress information. Signature: ``progress(op_code, cur_count, max_count=None, message='')``. - `Click here `__ for a description of all arguments - given to the function. - * An instance of a class derived from :class:`git.RemoteProgress` that - overrides the :meth:`~git.RemoteProgress.update` method. + See :meth:`RemoteProgress.update ` for a + description of all arguments given to the function. + * An instance of a class derived from :class:`~git.util.RemoteProgress` that + overrides the + :meth:`RemoteProgress.update ` method. :note: No further progress information is returned after push returns. From 1cd73ba118148e12b5dae7722efd5d86a640f64d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 24 Feb 2024 19:27:43 -0500 Subject: [PATCH 0670/1392] Revise docstrings in second-level modules Except for: - git.cmd, where docstrings were revised in e08066c. - git.types, where docstring changes may best be made together with changes to how imports are organized and documented, which seems not to be in the same scope as the changes in this commit. This change, as well as those in e08066c, are largely along the lines of #1725, with most revisions here being to docstrings and a few being to comments. The major differences between the kinds of docstring changes here and those ind #1725 are that the changes here push somewhat harder for consistency and apply some kinds of changes I was reluctant to apply widely in #1725: - Wrap all docstrings and comments to 88 columns, except for parts that are decisively clearer when not wrapped. Note that semi- paragraph changes represented as single newlines are still kept where meaningful, which is one reason this is not always the same effect as automatic wrapping would produce. - Avoid code formatting (double backticks) for headings that precede sections and code blocks. This was done enough that it seems to have been intentional, but it doesn't really have the right semantics, and the documentation is currently rendering in such a way (including on readthedocs.org) where removing that formatting seems clearly better. - References (single backticks with a role prefix) and code spans (double backticks) everywhere applicable, even in the first lines of docstrings. - Single-backticks around parameter names, with no role prefix. These were mostly either formatted that way or emphasized (with asterisks). This is one of the rare cases that I have used single backticks without a role prefix, which ordinarily should be avoided, but to get a role for references to a function's parameters within that function, a plugin would be needed. In the rare case that one function's docstring refers to another function's parameters by names those are double-backticked as code spans (and where applicable the name of the referred-to function is single-backticked with the :func: or :meth: role). - All sections, such as :param blah:, :note:, and :return:, now have a newline before any text in them. This was already often but far from always done, and the style was overall inconsistent. Of consistent approaches that are clear and easy to write, this is the simplest. It also seems to substantially improve readability, when taken together with... - Sections are always separated by a blank line, even if they are very short. - Essentially unlimited use of `~a.b.c`, where applicable, to refer and link to the documentation for a.b.c while showing the text "a" and revealing "a.b.c" on hover. I had previously somewhat limited my use of this tilde notation in case readers of the source code itself (where it is not rendered) weren't familiar with it, but at the cost of less consistency in when an entity was referred to. There remain a couple places in git.util where I do not do this because the explicit form `a `, which is equivalent, lined things up better and was thus easier to read. Those are the major differences between the approach taken here and in #1725, but not necessarily most of the changes done here (many of which are the same kinds of revisions as done there). Note that this commit only modifies some git/*.py files, and there are more git/**/*.py files that remain to be revised accordingly. --- git/compat.py | 15 +-- git/config.py | 153 ++++++++++++++++++----------- git/db.py | 9 +- git/diff.py | 64 +++++++----- git/exc.py | 9 +- git/remote.py | 262 ++++++++++++++++++++++++++++++++------------------ git/util.py | 188 ++++++++++++++++++++++-------------- 7 files changed, 439 insertions(+), 261 deletions(-) diff --git a/git/compat.py b/git/compat.py index 920e44b7f..3167cecdc 100644 --- a/git/compat.py +++ b/git/compat.py @@ -35,8 +35,9 @@ :attr:`sys.platform` checks explicitly, especially in cases where it matters which is used. -:note: ``is_win`` is ``False`` on Cygwin, but is often wrongly assumed ``True``. To - detect Cygwin, use ``sys.platform == "cygwin"``. +:note: + ``is_win`` is ``False`` on Cygwin, but is often wrongly assumed ``True``. To detect + Cygwin, use ``sys.platform == "cygwin"``. """ is_posix = os.name == "posix" @@ -46,9 +47,10 @@ :attr:`sys.platform` checks explicitly, especially in cases where it matters which is used. -:note: For POSIX systems, more detailed information is available in - :attr:`sys.platform`, while :attr:`os.name` is always ``"posix"`` on such systems, - including macOS (Darwin). +:note: + For POSIX systems, more detailed information is available in :attr:`sys.platform`, + while :attr:`os.name` is always ``"posix"`` on such systems, including macOS + (Darwin). """ is_darwin = sys.platform == "darwin" @@ -57,7 +59,8 @@ This is deprecated because it clearer to write out :attr:`os.name` or :attr:`sys.platform` checks explicitly. -:note: For macOS (Darwin), ``os.name == "posix"`` as in other Unix-like systems, while +:note: + For macOS (Darwin), ``os.name == "posix"`` as in other Unix-like systems, while ``sys.platform == "darwin"`. """ diff --git a/git/config.py b/git/config.py index 85f754197..b5cff01cd 100644 --- a/git/config.py +++ b/git/config.py @@ -73,11 +73,13 @@ class MetaParserBuilder(abc.ABCMeta): # noqa: B024 - """Utility class wrapping base-class methods into decorators that assure read-only properties.""" + """Utility class wrapping base-class methods into decorators that assure read-only + properties.""" def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> "MetaParserBuilder": """Equip all base-class methods with a needs_values decorator, and all non-const - methods with a set_dirty_and_flush_changes decorator in addition to that. + methods with a :func:`set_dirty_and_flush_changes` decorator in addition to + that. """ kmm = "_mutating_methods_" if kmm in clsdict: @@ -102,7 +104,8 @@ def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> "MetaParse def needs_values(func: Callable[..., _T]) -> Callable[..., _T]: - """Return a method for ensuring we read values (on demand) before we try to access them.""" + """Return a method for ensuring we read values (on demand) before we try to access + them.""" @wraps(func) def assure_data_present(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T: @@ -116,7 +119,8 @@ def assure_data_present(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _ def set_dirty_and_flush_changes(non_const_func: Callable[..., _T]) -> Callable[..., _T]: """Return a method that checks whether given non constant function may be called. - If so, the instance will be set dirty. Additionally, we flush the changes right to disk. + If so, the instance will be set dirty. Additionally, we flush the changes right to + disk. """ def flush_changes(self: "GitConfigParser", *args: Any, **kwargs: Any) -> _T: @@ -136,7 +140,8 @@ class SectionConstraint(Generic[T_ConfigParser]): It supports all ConfigParser methods that operate on an option. - :note: If used as a context manager, will release the wrapped ConfigParser. + :note: + If used as a context manager, will release the wrapped ConfigParser. """ __slots__ = ("_config", "_section_name") @@ -171,8 +176,8 @@ def __getattr__(self, attr: str) -> Any: return super().__getattribute__(attr) def _call_config(self, method: str, *args: Any, **kwargs: Any) -> Any: - """Call the configuration at the given method which must take a section name - as first argument.""" + """Call the configuration at the given method which must take a section name as + first argument.""" return getattr(self._config, method)(self._section_name, *args, **kwargs) @property @@ -254,7 +259,8 @@ def get_config_path(config_level: Lit_config_levels) -> str: elif config_level == "repository": raise ValueError("No repo to get repository configuration from. Use Repo._get_config_path") else: - # Should not reach here. Will raise ValueError if does. Static typing will warn missing elifs + # Should not reach here. Will raise ValueError if does. Static typing will warn + # about missing elifs. assert_never( # type: ignore[unreachable] config_level, ValueError(f"Invalid configuration level: {config_level!r}"), @@ -264,14 +270,15 @@ def get_config_path(config_level: Lit_config_levels) -> str: class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): """Implements specifics required to read git style configuration files. - This variation behaves much like the git.config command such that the configuration - will be read on demand based on the filepath given during initialization. + This variation behaves much like the ``git config`` command, such that the + configuration will be read on demand based on the filepath given during + initialization. The changes will automatically be written once the instance goes out of scope, but can be triggered manually as well. - The configuration file will be locked if you intend to change values preventing other - instances to write concurrently. + The configuration file will be locked if you intend to change values preventing + other instances to write concurrently. :note: The config is case-sensitive even when queried, hence section and option names @@ -301,7 +308,8 @@ class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): del optvalueonly_source _mutating_methods_ = ("add_section", "remove_section", "remove_option", "set") - """List of RawConfigParser methods able to change the instance.""" + """Names of :class:`~configparser.RawConfigParser` methods able to change the + instance.""" def __init__( self, @@ -311,8 +319,8 @@ def __init__( config_level: Union[Lit_config_levels, None] = None, repo: Union["Repo", None] = None, ) -> None: - """Initialize a configuration reader to read the given file_or_files and to - possibly allow changes to it by setting read_only False. + """Initialize a configuration reader to read the given `file_or_files` and to + possibly allow changes to it by setting `read_only` False. :param file_or_files: A file path or file object, or a sequence of possibly more than one of them. @@ -385,7 +393,7 @@ def _acquire_lock(self) -> None: # END read-only check def __del__(self) -> None: - """Write pending changes if required and release locks""" + """Write pending changes if required and release locks.""" # NOTE: Only consistent in Python 2. self.release() @@ -397,10 +405,12 @@ def __exit__(self, *args: Any) -> None: self.release() def release(self) -> None: - """Flush changes and release the configuration write lock. This instance must not be used anymore afterwards. + """Flush changes and release the configuration write lock. This instance must + not be used anymore afterwards. - In Python 3, it's required to explicitly release locks and flush changes, as __del__ is not called - deterministically anymore.""" + In Python 3, it's required to explicitly release locks and flush changes, as + :meth:`__del__` is not called deterministically anymore. + """ # Checking for the lock here makes sure we do not raise during write() # in case an invalid parser was created who could not get a lock. if self.read_only or (self._lock and not self._lock._has_lock()): @@ -424,8 +434,9 @@ def optionxform(self, optionstr: str) -> str: return optionstr def _read(self, fp: Union[BufferedReader, IO[bytes]], fpname: str) -> None: - """Originally a direct copy of the Python 2.4 version of RawConfigParser._read, - to ensure it uses ordered dicts. + """Originally a direct copy of the Python 2.4 version of + :meth:`RawConfigParser._read `, to ensure it + uses ordered dicts. The ordering bug was fixed in Python 2.4, and dict itself keeps ordering since Python 3.7. This has some other changes, especially that it ignores initial @@ -525,7 +536,8 @@ def _has_includes(self) -> Union[bool, int]: def _included_paths(self) -> List[Tuple[str, str]]: """List all paths that must be included to configuration. - :return: The list of paths, where each path is a tuple of ``(option, value)``. + :return: + The list of paths, where each path is a tuple of ``(option, value)``. """ paths = [] @@ -577,8 +589,11 @@ def read(self) -> None: # type: ignore[override] This will ignore files that cannot be read, possibly leaving an empty configuration. - :return: Nothing - :raise IOError: If a file cannot be handled + :return: + Nothing + + :raise IOError: + If a file cannot be handled """ if self._is_initialized: return @@ -591,7 +606,7 @@ def read(self) -> None: # type: ignore[override] elif not isinstance(self._file_or_files, (tuple, list, Sequence)): # Could merge with above isinstance once runtime type known. files_to_read = [self._file_or_files] - else: # for lists or tuples + else: # For lists or tuples. files_to_read = list(self._file_or_files) # END ensure we have a copy of the paths to handle @@ -603,7 +618,8 @@ def read(self) -> None: # type: ignore[override] if hasattr(file_path, "seek"): # Must be a file-object. - file_path = cast(IO[bytes], file_path) # TODO: Replace with assert to narrow type, once sure. + # TODO: Replace cast with assert to narrow type, once sure. + file_path = cast(IO[bytes], file_path) self._read(file_path, file_path.name) else: # Assume a path if it is not a file-object. @@ -615,8 +631,8 @@ def read(self) -> None: # type: ignore[override] except IOError: continue - # Read includes and append those that we didn't handle yet. - # We expect all paths to be normalized and absolute (and will ensure that is the case). + # Read includes and append those that we didn't handle yet. We expect all + # paths to be normalized and absolute (and will ensure that is the case). if self._has_includes(): for _, include_path in self._included_paths(): if include_path.startswith("~"): @@ -695,8 +711,9 @@ def items_all(self, section_name: str) -> List[Tuple[str, List[str]]]: def write(self) -> None: """Write changes to our file, if there are changes at all. - :raise IOError: If this is a read-only writer instance or if we could not obtain - a file lock""" + :raise IOError: + If this is a read-only writer instance or if we could not obtain a file lock + """ self._assure_writable("write") if not self._dirty: return @@ -740,7 +757,7 @@ def _assure_writable(self, method_name: str) -> None: raise IOError("Cannot execute non-constant method %s.%s" % (self, method_name)) def add_section(self, section: str) -> None: - """Assures added options will stay in order""" + """Assures added options will stay in order.""" return super().add_section(section) @property @@ -757,16 +774,18 @@ def get_value( ) -> Union[int, float, str, bool]: """Get an option's value. - If multiple values are specified for this option in the section, the - last one specified is returned. + If multiple values are specified for this option in the section, the last one + specified is returned. :param default: - If not None, the given default value will be returned in case - the option did not exist + If not None, the given default value will be returned in case the option did + not exist - :return: a properly typed value, either int, float or string + :return: + A properly typed value, either int, float or string - :raise TypeError: in case the value could not be understood + :raise TypeError: + In case the value could not be understood. Otherwise the exceptions known to the ConfigParser will be raised. """ try: @@ -790,12 +809,14 @@ def get_values( returned. :param default: - If not None, a list containing the given default value will be - returned in case the option did not exist + If not None, a list containing the given default value will be returned in + case the option did not exist. - :return: a list of properly typed values, either int, float or string + :return: + A list of properly typed values, either int, float or string - :raise TypeError: in case the value could not be understood + :raise TypeError: + In case the value could not be understood. Otherwise the exceptions known to the ConfigParser will be raised. """ try: @@ -849,11 +870,17 @@ def set_value(self, section: str, option: str, value: Union[str, bytes, int, flo This will create the section if required, and will not throw as opposed to the default ConfigParser 'set' method. - :param section: Name of the section in which the option resides or should reside - :param option: Name of the options whose value to set - :param value: Value to set the option to. It must be a string or convertible to - a string. - :return: This instance + :param section: + Name of the section in which the option resides or should reside. + + :param option: + Name of the options whose value to set. + + :param value: + Value to set the option to. It must be a string or convertible to a string. + + :return: + This instance """ if not self.has_section(section): self.add_section(section) @@ -865,15 +892,22 @@ def set_value(self, section: str, option: str, value: Union[str, bytes, int, flo def add_value(self, section: str, option: str, value: Union[str, bytes, int, float, bool]) -> "GitConfigParser": """Add a value for the given option in section. - This will create the section if required, and will not throw as opposed to the default - ConfigParser 'set' method. The value becomes the new value of the option as returned - by 'get_value', and appends to the list of values returned by 'get_values`'. + This will create the section if required, and will not throw as opposed to the + default ConfigParser 'set' method. The value becomes the new value of the option + as returned by 'get_value', and appends to the list of values returned by + 'get_values'. - :param section: Name of the section in which the option resides or should reside - :param option: Name of the option - :param value: Value to add to option. It must be a string or convertible - to a string - :return: This instance + :param section: + Name of the section in which the option resides or should reside. + + :param option: + Name of the option. + + :param value: + Value to add to option. It must be a string or convertible to a string. + + :return: + This instance """ if not self.has_section(section): self.add_section(section) @@ -883,8 +917,12 @@ def add_value(self, section: str, option: str, value: Union[str, bytes, int, flo def rename_section(self, section: str, new_name: str) -> "GitConfigParser": """Rename the given section to new_name. - :raise ValueError: If ``section`` doesn't exist - :raise ValueError: If a section with ``new_name`` does already exist + :raise ValueError: + If: + + * ``section`` doesn't exist. + * A section with ``new_name`` does already exist. + :return: This instance """ if not self.has_section(section): @@ -898,6 +936,7 @@ def rename_section(self, section: str, new_name: str) -> "GitConfigParser": new_section.setall(k, vs) # END for each value to copy - # This call writes back the changes, which is why we don't have the respective decorator. + # This call writes back the changes, which is why we don't have the respective + # decorator. self.remove_section(section) return self diff --git a/git/db.py b/git/db.py index 1531d663c..dff59f47a 100644 --- a/git/db.py +++ b/git/db.py @@ -31,8 +31,9 @@ class GitCmdObjectDB(LooseObjectDB): It will create objects only in the loose object database. - :note: For now, we use the git command to do all the lookup, just until we - have packs and the other implementations. + :note: + For now, we use the git command to do all the lookup, just until we have packs + and the other implementations. """ def __init__(self, root_path: PathLike, git: "Git") -> None: @@ -56,9 +57,11 @@ def partial_to_complete_sha_hex(self, partial_hexsha: str) -> bytes: :return: Full binary 20 byte sha from the given partial hexsha :raise AmbiguousObjectName: + :raise BadObject: - :note: Currently we only raise :class:`BadObject` as git does not communicate + :note: + Currently we only raise :class:`BadObject` as git does not communicate ambiguous objects separately. """ try: diff --git a/git/diff.py b/git/diff.py index aba1a1080..7205136d0 100644 --- a/git/diff.py +++ b/git/diff.py @@ -84,8 +84,9 @@ class Diffable: compatible type. :note: - Subclasses require a repo member as it is the case for Object instances, for - practical reasons we do not derive from Object. + Subclasses require a repo member as it is the case for + :class:`~git.objects.base.Object` instances, for practical reasons we do not + derive from :class:`~git.objects.base.Object`. """ __slots__ = () @@ -135,13 +136,13 @@ def diff( to be read and diffed. :param kwargs: - Additional arguments passed to git-diff, such as ``R=True`` to swap both + Additional arguments passed to ``git diff``, such as ``R=True`` to swap both sides of the diff. :return: git.DiffIndex :note: - On a bare repository, 'other' needs to be provided as + On a bare repository, `other` needs to be provided as :class:`~Diffable.Index`, or as :class:`~git.objects.tree.Tree` or :class:`~git.objects.commit.Commit`, or a git command error will occur. """ @@ -183,7 +184,7 @@ def diff( args.insert(0, self) - # paths is list here, or None. + # paths is a list here, or None. if paths: args.append("--") args.extend(paths) @@ -203,7 +204,7 @@ def diff( class DiffIndex(List[T_Diff]): - """An Index for diffs, allowing a list of Diffs to be queried by the diff + R"""An Index for diffs, allowing a list of :class:`Diff`\s to be queried by the diff properties. The class improves the diff handling convenience. @@ -255,27 +256,27 @@ def iter_change_type(self, change_type: Lit_change_type) -> Iterator[T_Diff]: class Diff: """A Diff contains diff information between two Trees. - It contains two sides a and b of the diff, members are prefixed with - "a" and "b" respectively to indicate that. + It contains two sides a and b of the diff. Members are prefixed with "a" and "b" + respectively to indicate that. Diffs keep information about the changed blob objects, the file mode, renames, deletions and new files. There are a few cases where None has to be expected as member variable value: - ``New File``:: + New File:: a_mode is None a_blob is None a_path is None - ``Deleted File``:: + Deleted File:: b_mode is None b_blob is None b_path is None - ``Working Tree Blobs`` + Working Tree Blobs: When comparing to working trees, the working tree blob will have a null hexsha as a corresponding object does not yet exist. The mode will be null as well. @@ -469,7 +470,8 @@ def renamed(self) -> bool: """ :return: True if the blob of our diff has been renamed - :note: This property is deprecated. + :note: + This property is deprecated. Please use the :attr:`renamed_file` property instead. """ return self.renamed_file @@ -494,11 +496,17 @@ def _pick_best_path(cls, path_match: bytes, rename_match: bytes, path_fallback_m @classmethod def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoInterrupt"]) -> DiffIndex: - """Create a new DiffIndex from the given process output which must be in patch format. + """Create a new :class:`DiffIndex` from the given process output which must be + in patch format. - :param repo: The repository we are operating on - :param proc: ``git diff`` process to read from (supports :class:`Git.AutoInterrupt` wrapper) - :return: git.DiffIndex + :param repo: The repository we are operating on. + + :param proc: + ``git diff`` process to read from + (supports :class:`Git.AutoInterrupt` wrapper). + + :return: + :class:`DiffIndex` """ # FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise. @@ -539,14 +547,14 @@ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoIn a_path = cls._pick_best_path(a_path, rename_from, a_path_fallback) b_path = cls._pick_best_path(b_path, rename_to, b_path_fallback) - # Our only means to find the actual text is to see what has not been matched by our regex, - # and then retro-actively assign it to our index. + # Our only means to find the actual text is to see what has not been matched + # by our regex, and then retro-actively assign it to our index. if previous_header is not None: index[-1].diff = text[previous_header.end() : _header.start()] # END assign actual diff - # Make sure the mode is set if the path is set. Otherwise the resulting blob is invalid. - # We just use the one mode we should have parsed. + # Make sure the mode is set if the path is set. Otherwise the resulting blob + # is invalid. We just use the one mode we should have parsed. a_mode = old_mode or deleted_file_mode or (a_path and (b_mode or new_mode or new_file_mode)) b_mode = b_mode or new_mode or new_file_mode or (b_path and a_mode) index.append( @@ -610,7 +618,7 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non rename_from = None rename_to = None - # NOTE: We cannot conclude from the existence of a blob to change type + # NOTE: We cannot conclude from the existence of a blob to change type, # as diffs with the working do not have blobs yet. if change_type == "D": b_blob_id = None # Optional[str] @@ -654,11 +662,17 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non @classmethod def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex": - """Create a new DiffIndex from the given process output which must be in raw format. + """Create a new :class:`DiffIndex` from the given process output which must be + in raw format. - :param repo: The repository we are operating on - :param proc: Process to read output from - :return: git.DiffIndex + :param repo: + The repository we are operating on. + + :param proc: + Process to read output from. + + :return: + :class:`DiffIndex` """ # handles # :100644 100644 687099101... 37c5e30c8... M .gitignore diff --git a/git/exc.py b/git/exc.py index 40fb8eea0..276d79e42 100644 --- a/git/exc.py +++ b/git/exc.py @@ -81,7 +81,8 @@ class UnsafeOptionError(GitError): class CommandError(GitError): - """Base class for exceptions thrown at every stage of `Popen()` execution. + """Base class for exceptions thrown at every stage of :class:`~subprocess.Popen` + execution. :param command: A non-empty list of argv comprising the command-line. @@ -135,8 +136,8 @@ def __str__(self) -> str: class GitCommandNotFound(CommandError): - """Thrown if we cannot find the `git` executable in the PATH or at the path given by - the GIT_PYTHON_GIT_EXECUTABLE environment variable.""" + """Thrown if we cannot find the `git` executable in the ``PATH`` or at the path + given by the ``GIT_PYTHON_GIT_EXECUTABLE`` environment variable.""" def __init__(self, command: Union[List[str], Tuple[str], str], cause: Union[str, Exception]) -> None: super().__init__(command, cause) @@ -187,7 +188,7 @@ def __str__(self) -> str: class CacheError(GitError): - """Base for all errors related to the git index, which is called cache + """Base for all errors related to the git index, which is called "cache" internally.""" diff --git a/git/remote.py b/git/remote.py index df809a9ac..5bee9edf5 100644 --- a/git/remote.py +++ b/git/remote.py @@ -70,13 +70,15 @@ def add_progress( git: Git, progress: Union[RemoteProgress, "UpdateProgress", Callable[..., RemoteProgress], None], ) -> Any: - """Add the --progress flag to the given kwargs dict if supported by the - git command. + """Add the ``--progress`` flag to the given `kwargs` dict if supported by the git + command. - :note: If the actual progress in the given progress instance is not - given, we do not request any progress. + :note: + If the actual progress in the given progress instance is not given, we do not + request any progress. - :return: possibly altered kwargs + :return: + Possibly altered `kwargs` """ if progress is not None: v = git.version_info[:2] @@ -108,7 +110,8 @@ def to_progress_instance(progress: RemoteProgress) -> RemoteProgress: def to_progress_instance( progress: Union[Callable[..., Any], RemoteProgress, None] ) -> Union[RemoteProgress, CallableRemoteProgress]: - """Given the 'progress' return a suitable object derived from RemoteProgress.""" + """Given the `progress` return a suitable object derived from + :class:`~git.util.RemoteProgress`.""" # New API only needs progress as a function. if callable(progress): return CallableRemoteProgress(progress) @@ -276,7 +279,7 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> NoReturn: # -> class PushInfoList(IterableList[PushInfo]): - """IterableList of PushInfo objects.""" + """IterableList of :class:`PushInfo` objects.""" def __new__(cls) -> "PushInfoList": return cast(PushInfoList, IterableList.__new__(cls, "push_infos")) @@ -380,8 +383,8 @@ def commit(self) -> Commit_ish: @classmethod def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": - """Parse information from the given line as returned by git-fetch -v - and return a new FetchInfo object representing this information. + """Parse information from the given line as returned by ``git-fetch -v`` and + return a new :class:`FetchInfo` object representing this information. We can handle a line as follows: "%c %-\\*s %-\\*s -> %s%s" @@ -391,7 +394,7 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": + means success forcing update - means a tag was updated * means birth of new branch or tag - = means the head was up to date ( and not moved ) + = means the head was up to date (and not moved) ' ' means a fast-forward fetch line is the corresponding line from FETCH_HEAD, like @@ -455,15 +458,17 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": if remote_local_ref_str == "FETCH_HEAD": ref_type = SymbolicReference elif ref_type_name == "tag" or is_tag_operation: - # The ref_type_name can be branch, whereas we are still seeing a tag operation. - # It happens during testing, which is based on actual git operations. + # The ref_type_name can be branch, whereas we are still seeing a tag + # operation. It happens during testing, which is based on actual git + # operations. ref_type = TagReference elif ref_type_name in ("remote-tracking", "branch"): - # Note: remote-tracking is just the first part of the 'remote-tracking branch' token. - # We don't parse it correctly, but its enough to know what to do, and it's new in git 1.7something. + # Note: remote-tracking is just the first part of the + # 'remote-tracking branch' token. We don't parse it correctly, but it's + # enough to know what to do, and it's new in git 1.7something. ref_type = RemoteReference elif "/" in ref_type_name: - # If the fetch spec look something like this '+refs/pull/*:refs/heads/pull/*', + # If the fetch spec look something like '+refs/pull/*:refs/heads/pull/*', # and is thus pretty much anything the user wants, we will have trouble # determining what's going on. For now, we assume the local ref is a Head. ref_type = Head @@ -475,17 +480,19 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": if ref_type is SymbolicReference: remote_local_ref = ref_type(repo, "FETCH_HEAD") else: - # Determine prefix. Tags are usually pulled into refs/tags, they may have subdirectories. - # It is not clear sometimes where exactly the item is, unless we have an absolute path as - # indicated by the 'ref/' prefix. Otherwise even a tag could be in refs/remotes, which is - # when it will have the 'tags/' subdirectory in its path. - # We don't want to test for actual existence, but try to figure everything out analytically. + # Determine prefix. Tags are usually pulled into refs/tags; they may have + # subdirectories. It is not clear sometimes where exactly the item is, + # unless we have an absolute path as indicated by the 'ref/' prefix. + # Otherwise even a tag could be in refs/remotes, which is when it will have + # the 'tags/' subdirectory in its path. We don't want to test for actual + # existence, but try to figure everything out analytically. ref_path: Optional[PathLike] = None remote_local_ref_str = remote_local_ref_str.strip() if remote_local_ref_str.startswith(Reference._common_path_default + "/"): - # Always use actual type if we get absolute paths. - # Will always be the case if something is fetched outside of refs/remotes (if its not a tag). + # Always use actual type if we get absolute paths. This will always be + # the case if something is fetched outside of refs/remotes (if its not a + # tag). ref_path = remote_local_ref_str if ref_type is not TagReference and not remote_local_ref_str.startswith( RemoteReference._common_path_default + "/" @@ -499,8 +506,8 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": ref_path = join_path(ref_type._common_path_default, remote_local_ref_str) # END obtain refpath - # Even though the path could be within the git conventions, we make - # sure we respect whatever the user wanted, and disabled path checking. + # Even though the path could be within the git conventions, we make sure we + # respect whatever the user wanted, and disabled path checking. remote_local_ref = ref_type(repo, ref_path, check_path=False) # END create ref instance @@ -517,10 +524,11 @@ class Remote(LazyMixin, IterableObj): """Provides easy read and write access to a git remote. Everything not part of this interface is considered an option for the current - remote, allowing constructs like remote.pushurl to query the pushurl. + remote, allowing constructs like ``remote.pushurl`` to query the pushurl. - :note: When querying configuration, the configuration accessor will be cached - to speed up subsequent accesses. + :note: + When querying configuration, the configuration accessor will be cached to speed + up subsequent accesses. """ __slots__ = ("repo", "name", "_config_reader") @@ -547,21 +555,24 @@ class Remote(LazyMixin, IterableObj): def __init__(self, repo: "Repo", name: str) -> None: """Initialize a remote instance. - :param repo: The repository we are a remote of - :param name: The name of the remote, e.g. 'origin' + :param repo: + The repository we are a remote of. + + :param name: + The name of the remote, e.g. 'origin'. """ self.repo = repo self.name = name self.url: str def __getattr__(self, attr: str) -> Any: - """Allows to call this instance like - remote.special( \\*args, \\*\\*kwargs) to call git-remote special self.name.""" + """Allows to call this instance like ``remote.special(*args, **kwargs)`` to + call ``git remote special self.name``.""" if attr == "_config_reader": return super().__getattr__(attr) - # Sometimes, probably due to a bug in Python itself, we are being called - # even though a slot of the same name exists. + # Sometimes, probably due to a bug in Python itself, we are being called even + # though a slot of the same name exists. try: return self._config_reader.get(attr) except cp.NoOptionError: @@ -599,7 +610,8 @@ def __hash__(self) -> int: def exists(self) -> bool: """ - :return: True if this is a valid, existing remote. + :return: + True if this is a valid, existing remote. Valid remotes have an entry in the repository's configuration. """ try: @@ -627,14 +639,21 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator["Remote def set_url( self, new_url: str, old_url: Optional[str] = None, allow_unsafe_protocols: bool = False, **kwargs: Any ) -> "Remote": - """Configure URLs on current remote (cf command git remote set_url). + """Configure URLs on current remote (cf command ``git remote set-url``). This command manages URLs on the remote. - :param new_url: String being the URL to add as an extra remote URL - :param old_url: When set, replaces this URL with new_url for the remote - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :return: self + :param new_url: + String being the URL to add as an extra remote URL. + + :param old_url: + When set, replaces this URL with `new_url` for the remote. + + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :return: + self """ if not allow_unsafe_protocols: Git.check_unsafe_protocols(new_url) @@ -647,25 +666,33 @@ def set_url( return self def add_url(self, url: str, allow_unsafe_protocols: bool = False, **kwargs: Any) -> "Remote": - """Adds a new url on current remote (special case of git remote set_url). + """Adds a new url on current remote (special case of ``git remote set-url``). This command adds new URLs to a given remote, making it possible to have multiple URLs for a single remote. - :param url: String being the URL to add as an extra remote URL - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :return: self + :param url: + String being the URL to add as an extra remote URL. + + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :return: + self """ return self.set_url(url, add=True, allow_unsafe_protocols=allow_unsafe_protocols) def delete_url(self, url: str, **kwargs: Any) -> "Remote": - """Deletes a new url on current remote (special case of git remote set_url) + """Deletes a new url on current remote (special case of ``git remote set-url``) This command deletes new URLs to a given remote, making it possible to have multiple URLs for a single remote. - :param url: String being the URL to delete from the remote - :return: self + :param url: + String being the URL to delete from the remote. + + :return: + self """ return self.set_url(url, delete=True) @@ -706,9 +733,12 @@ def urls(self) -> Iterator[str]: def refs(self) -> IterableList[RemoteReference]: """ :return: - IterableList of RemoteReference objects. It is prefixed, allowing - you to omit the remote path portion, e.g.:: - remote.refs.master # yields RemoteReference('/refs/remotes/origin/master') + :class:`~git.util.IterableList` of :class:`git.refs.remote.RemoteReference` + objects. + + It is prefixed, allowing you to omit the remote path portion, e.g.:: + + remote.refs.master # yields RemoteReference('/refs/remotes/origin/master') """ out_refs: IterableList[RemoteReference] = IterableList(RemoteReference._id_attribute_, "%s/" % self.name) out_refs.extend(RemoteReference.list_items(self.repo, remote=self.name)) @@ -718,12 +748,13 @@ def refs(self) -> IterableList[RemoteReference]: def stale_refs(self) -> IterableList[Reference]: """ :return: - IterableList RemoteReference objects that do not have a corresponding - head in the remote reference anymore as they have been deleted on the - remote side, but are still available locally. + :class:`~git.util.IterableList` of :class:`git.refs.remote.RemoteReference` + objects that do not have a corresponding head in the remote reference + anymore as they have been deleted on the remote side, but are still + available locally. - The IterableList is prefixed, hence the 'origin' must be omitted. See - 'refs' property for an example. + The :class:`~git.util.IterableList` is prefixed, hence the 'origin' must be + omitted. See :attr:`refs` property for an example. To make things more complicated, it can be possible for the list to include other kinds of references, for example, tag references, if these are stale @@ -752,13 +783,26 @@ def stale_refs(self) -> IterableList[Reference]: def create(cls, repo: "Repo", name: str, url: str, allow_unsafe_protocols: bool = False, **kwargs: Any) -> "Remote": """Create a new remote to the given repository. - :param repo: Repository instance that is to receive the new remote - :param name: Desired name of the remote - :param url: URL which corresponds to the remote's name - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :param kwargs: Additional arguments to be passed to the git-remote add command - :return: New Remote instance - :raise GitCommandError: in case an origin with that name already exists + :param repo: + Repository instance that is to receive the new remote. + + :param name: + Desired name of the remote. + + :param url: + URL which corresponds to the remote's name. + + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :param kwargs: + Additional arguments to be passed to the ``git remote add`` command. + + :return: + New :class:`Remote` instance + + :raise GitCommandError: + In case an origin with that name already exists. """ scmd = "add" kwargs["insert_kwargs_after"] = scmd @@ -777,7 +821,8 @@ def add(cls, repo: "Repo", name: str, url: str, **kwargs: Any) -> "Remote": def remove(cls, repo: "Repo", name: str) -> str: """Remove the remote with the given name. - :return: The passed remote name to remove + :return: + The passed remote name to remove """ repo.git.remote("rm", name) if isinstance(name, cls): @@ -788,9 +833,10 @@ def remove(cls, repo: "Repo", name: str) -> str: rm = remove def rename(self, new_name: str) -> "Remote": - """Rename self to the given new_name. + """Rename self to the given `new_name`. - :return: self + :return: + self """ if self.name == new_name: return self @@ -802,12 +848,15 @@ def rename(self, new_name: str) -> "Remote": return self def update(self, **kwargs: Any) -> "Remote": - """Fetch all changes for this remote, including new branches which will - be forced in (in case your local remote branch is not part the new remote - branch's ancestry anymore). + """Fetch all changes for this remote, including new branches which will be + forced in (in case your local remote branch is not part the new remote branch's + ancestry anymore). + + :param kwargs: + Additional arguments passed to ``git remote update``. - :param kwargs: Additional arguments passed to git-remote update - :return: self + :return: + self """ scmd = "update" kwargs["insert_kwargs_after"] = scmd @@ -966,9 +1015,9 @@ def fetch( Taken from the git manual, gitglossary(7). - Fetch supports multiple refspecs (as the - underlying git-fetch does) - supplying a list rather than a string - for 'refspec' will make use of this facility. + Fetch supports multiple refspecs (as the underlying git-fetch does) - + supplying a list rather than a string for 'refspec' will make use of this + facility. :param progress: See :meth:`push` method. @@ -978,15 +1027,18 @@ def fetch( To specify a timeout in seconds for the git command, after which the process should be killed. It is set to None by default. - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack. + :param allow_unsafe_options: + Allow unsafe options to be used, like ``--upload-pack``. - :param kwargs: Additional arguments to be passed to git-fetch. + :param kwargs: + Additional arguments to be passed to ``git fetch``. :return: - IterableList(FetchInfo, ...) list of FetchInfo instances providing detailed - information about the fetch results + IterableList(FetchInfo, ...) list of :class:`FetchInfo` instances providing + detailed information about the fetch results :note: As fetch does not provide progress information to non-ttys, we cannot make @@ -1030,13 +1082,26 @@ def pull( """Pull changes from the given branch, being the same as a fetch followed by a merge of branch with your local branch. - :param refspec: See :meth:`fetch` method - :param progress: See :meth:`push` method - :param kill_after_timeout: See :meth:`fetch` method - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack - :param kwargs: Additional arguments to be passed to git-pull - :return: Please see :meth:`fetch` method + :param refspec: + See :meth:`fetch` method. + + :param progress: + See :meth:`push` method. + + :param kill_after_timeout: + See :meth:`fetch` method. + + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :param allow_unsafe_options: + Allow unsafe options to be used, like ``--upload-pack``. + + :param kwargs: + Additional arguments to be passed to ``git pull``. + + :return: + Please see :meth:`fetch` method """ if refspec is None: # No argument refspec, then ensure the repo's config has a fetch refspec. @@ -1070,7 +1135,8 @@ def push( ) -> PushInfoList: """Push changes from source branch in refspec to target branch in refspec. - :param refspec: See :meth:`fetch` method. + :param refspec: + See :meth:`fetch` method. :param progress: Can take one of many value types: @@ -1084,26 +1150,31 @@ def push( overrides the :meth:`RemoteProgress.update ` method. - :note: No further progress information is returned after push returns. + :note: + No further progress information is returned after push returns. :param kill_after_timeout: To specify a timeout in seconds for the git command, after which the process should be killed. It is set to None by default. - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. :param allow_unsafe_options: - Allow unsafe options to be used, like --receive-pack. + Allow unsafe options to be used, like ``--receive-pack``. - :param kwargs: Additional arguments to be passed to git-push. + :param kwargs: Additional arguments to be passed to ``git push``. :return: A :class:`PushInfoList` object, where each list member represents an individual head which had been updated on the remote side. + If the push contains rejected heads, these will have the :attr:`PushInfo.ERROR` bit set in their flags. - If the operation fails completely, the length of the returned PushInfoList - will be 0. + + If the operation fails completely, the length of the returned + :class:`PushInfoList` will be 0. + Call :meth:`~PushInfoList.raise_if_error` on the returned object to raise on any failure. """ @@ -1133,8 +1204,9 @@ def push( def config_reader(self) -> SectionConstraint[GitConfigParser]: """ :return: - GitConfigParser compatible object able to read options for only our remote. - Hence you may simple type config.get("pushurl") to obtain the information. + :class:`~git.config.GitConfigParser` compatible object able to read options + for only our remote. Hence you may simply type ``config.get("pushurl")`` to + obtain the information. """ return self._config_reader @@ -1148,7 +1220,9 @@ def _clear_cache(self) -> None: @property def config_writer(self) -> SectionConstraint: """ - :return: GitConfigParser compatible object able to write options for this remote. + :return: + :class:`~git.config.GitConfigParser`-compatible object able to write options + for this remote. :note: You can only own one writer at a time - delete it to release the diff --git a/git/util.py b/git/util.py index 03d62ffc3..30b78f7b2 100644 --- a/git/util.py +++ b/git/util.py @@ -111,10 +111,11 @@ def _read_win_env_flag(name: str, default: bool) -> bool: """Read a boolean flag from an environment variable on Windows. :return: - On Windows, the flag, or the ``default`` value if absent or ambiguous. - On all other operating systems, ``False``. + On Windows, the flag, or the `default` value if absent or ambiguous. + On all other operating systems, False. - :note: This only accesses the environment on Windows. + :note: + This only accesses the environment on Windows. """ if os.name != "nt": return False @@ -151,8 +152,8 @@ def _read_win_env_flag(name: str, default: bool) -> bool: def unbare_repo(func: Callable[..., T]) -> Callable[..., T]: - """Methods with this decorator raise :class:`.exc.InvalidGitRepositoryError` if they - encounter a bare repository.""" + """Methods with this decorator raise + :class:`~git.exc.InvalidGitRepositoryError` if they encounter a bare repository.""" from .exc import InvalidGitRepositoryError @@ -206,7 +207,10 @@ def rmtree(path: PathLike) -> None: """ def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: - """Callback for :func:`shutil.rmtree`. Works either as ``onexc`` or ``onerror``.""" + """Callback for :func:`shutil.rmtree`. + + This works as either a ``onexc`` or ``onerror`` style callback. + """ # Is the error an access error? os.chmod(path, stat.S_IWUSR) @@ -228,7 +232,8 @@ def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: def rmfile(path: PathLike) -> None: - """Ensure file deleted also on *Windows* where read-only files need special treatment.""" + """Ensure file deleted also on *Windows* where read-only files need special + treatment.""" if osp.isfile(path): if os.name == "nt": os.chmod(path, 0o777) @@ -239,7 +244,8 @@ def stream_copy(source: BinaryIO, destination: BinaryIO, chunk_size: int = 512 * """Copy all data from the source stream into the destination stream in chunks of size chunk_size. - :return: Number of bytes written + :return: + Number of bytes written """ br = 0 while True: @@ -473,12 +479,13 @@ def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: def get_user_id() -> str: - """:return: string identifying the currently active system user as name@node""" + """:return: String identifying the currently active system user as ``name@node``""" return "%s@%s" % (getpass.getuser(), platform.node()) def finalize_process(proc: Union[subprocess.Popen, "Git.AutoInterrupt"], **kwargs: Any) -> None: - """Wait for the process (clone, fetch, pull or push) and handle its errors accordingly""" + """Wait for the process (clone, fetch, pull or push) and handle its errors + accordingly.""" # TODO: No close proc-streams?? proc.wait(**kwargs) @@ -541,9 +548,9 @@ def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: class RemoteProgress: - """ - Handler providing an interface to parse progress information emitted by git-push - and git-fetch and to dispatch callbacks allowing subclasses to react to the progress. + """Handler providing an interface to parse progress information emitted by + ``git push`` and ``git fetch`` and to dispatch callbacks allowing subclasses to + react to the progress. """ _num_op_codes: int = 9 @@ -580,8 +587,8 @@ def __init__(self) -> None: self.other_lines: List[str] = [] def _parse_progress_line(self, line: AnyStr) -> None: - """Parse progress information from the given line as retrieved by git-push - or git-fetch. + """Parse progress information from the given line as retrieved by ``git push`` + or ``git fetch``. - Lines that do not contain progress info are stored in :attr:`other_lines`. - Lines that seem to contain an error (i.e. start with ``error:`` or ``fatal:``) @@ -685,8 +692,8 @@ def _parse_progress_line(self, line: AnyStr) -> None: def new_message_handler(self) -> Callable[[str], None]: """ :return: - A progress handler suitable for handle_process_output(), passing lines on to - this Progress handler in a suitable format + A progress handler suitable for :func:`~git.cmd.handle_process_output`, + passing lines on to this progress handler in a suitable format. """ def handler(line: AnyStr) -> None: @@ -712,25 +719,29 @@ def update( :param op_code: Integer allowing to be compared against Operation IDs and stage IDs. - Stage IDs are BEGIN and END. BEGIN will only be set once for each Operation - ID as well as END. It may be that BEGIN and END are set at once in case only - one progress message was emitted due to the speed of the operation. - Between BEGIN and END, none of these flags will be set. + Stage IDs are :attr:`BEGIN` and :attr:`END`. :attr:`BEGIN` will only be set + once for each Operation ID as well as :attr:`END`. It may be that + :attr:`BEGIN` and :attr:`END` are set at once in case only one progress + message was emitted due to the speed of the operation. Between :attr:`BEGIN` + and :attr:`END`, none of these flags will be set. - Operation IDs are all held within the OP_MASK. Only one Operation ID will - be active per call. + Operation IDs are all held within the :attr:`OP_MASK`. Only one Operation ID + will be active per call. - :param cur_count: Current absolute count of items. + :param cur_count: + Current absolute count of items. :param max_count: - The maximum count of items we expect. It may be None in case there is - no maximum number of items or if it is (yet) unknown. + The maximum count of items we expect. It may be None in case there is no + maximum number of items or if it is (yet) unknown. :param message: - In case of the 'WRITING' operation, it contains the amount of bytes + In case of the :attr:`WRITING` operation, it contains the amount of bytes transferred. It may possibly be used for other purposes as well. - You may read the contents of the current line in ``self._cur_line``. + :note: + You may read the contents of the current line in + :attr:`self._cur_line <_cur_line>`. """ pass @@ -793,11 +804,13 @@ def __repr__(self) -> str: def _from_string(cls, string: str) -> "Actor": """Create an Actor from a string. - :param string: The string, which is expected to be in regular git format:: + :param string: + The string, which is expected to be in regular git format:: - John Doe + John Doe - :return: Actor + :return: + :class:`Actor` """ m = cls.name_email_regex.search(string) if m: @@ -857,18 +870,19 @@ def committer(cls, config_reader: Union[None, "GitConfigParser", "SectionConstra """ :return: Actor instance corresponding to the configured committer. It behaves similar to the git implementation, such that the environment will override - configuration values of config_reader. If no value is set at all, it will be - generated. + configuration values of `config_reader`. If no value is set at all, it will + be generated. - :param config_reader: ConfigReader to use to retrieve the values from in case - they are not set in the environment. + :param config_reader: + ConfigReader to use to retrieve the values from in case they are not set in + the environment. """ return cls._main_actor(cls.env_committer_name, cls.env_committer_email, config_reader) @classmethod def author(cls, config_reader: Union[None, "GitConfigParser", "SectionConstraint"] = None) -> "Actor": - """Same as committer(), but defines the main author. It may be specified in the - environment, but defaults to the committer.""" + """Same as :meth:`committer`, but defines the main author. It may be specified + in the environment, but defaults to the committer.""" return cls._main_actor(cls.env_author_name, cls.env_author_email, config_reader) @@ -877,7 +891,7 @@ class Stats: Represents stat information as presented by git at the end of a merge. It is created from the output of a diff operation. - ``Example``:: + Example:: c = Commit( sha1 ) s = c.stats @@ -907,9 +921,10 @@ def __init__(self, total: Total_TD, files: Dict[PathLike, Files_TD]): @classmethod def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": - """Create a Stat object from output retrieved by git-diff. + """Create a :class:`Stats` object from output retrieved by ``git diff``. - :return: git.Stat + :return: + :class:`git.Stats` """ hsh: HSH_TD = { @@ -936,11 +951,12 @@ def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": class IndexFileSHA1Writer: """Wrapper around a file-like object that remembers the SHA1 of the data written to it. It will write a sha when the stream is closed - or if the asked for explicitly using write_sha. + or if asked for explicitly using :meth:`write_sha`. Only useful to the index file. - :note: Based on the dulwich project. + :note: + Based on the dulwich project. """ __slots__ = ("f", "sha1") @@ -991,16 +1007,20 @@ def _lock_file_path(self) -> str: def _has_lock(self) -> bool: """ - :return: True if we have a lock and if the lockfile still exists + :return: + True if we have a lock and if the lockfile still exists - :raise AssertionError: If our lock-file does not exist + :raise AssertionError: + If our lock-file does not exist. """ return self._owns_lock def _obtain_lock_or_raise(self) -> None: - """Create a lock file as flag for other instances, mark our instance as lock-holder. + """Create a lock file as flag for other instances, mark our instance as + lock-holder. - :raise IOError: If a lock was already present or a lock file could not be written + :raise IOError: + If a lock was already present or a lock file could not be written. """ if self._has_lock(): return @@ -1021,7 +1041,9 @@ def _obtain_lock_or_raise(self) -> None: def _obtain_lock(self) -> None: """The default implementation will raise if a lock cannot be obtained. - Subclasses may override this method to provide a different implementation.""" + + Subclasses may override this method to provide a different implementation. + """ return self._obtain_lock_or_raise() def _release_lock(self) -> None: @@ -1029,8 +1051,8 @@ def _release_lock(self) -> None: if not self._has_lock(): return - # If someone removed our file beforehand, lets just flag this issue - # instead of failing, to make it more usable. + # If someone removed our file beforehand, lets just flag this issue instead of + # failing, to make it more usable. lfp = self._lock_file_path() try: rmfile(lfp) @@ -1040,12 +1062,12 @@ def _release_lock(self) -> None: class BlockingLockFile(LockFile): - """The lock file will block until a lock could be obtained, or fail after - a specified timeout. + """The lock file will block until a lock could be obtained, or fail after a + specified timeout. - :note: If the directory containing the lock was removed, an exception will - be raised during the blocking period, preventing hangs as the lock - can never be obtained. + :note: + If the directory containing the lock was removed, an exception will be raised + during the blocking period, preventing hangs as the lock can never be obtained. """ __slots__ = ("_check_interval", "_max_block_time") @@ -1062,14 +1084,15 @@ def __init__( Period of time to sleep until the lock is checked the next time. By default, it waits a nearly unlimited time. - :param max_block_time_s: Maximum amount of seconds we may lock. + :param max_block_time_s: + Maximum amount of seconds we may lock. """ super().__init__(file_path) self._check_interval = check_interval_s self._max_block_time = max_block_time_s def _obtain_lock(self) -> None: - """This method blocks until it obtained the lock, or raises IOError if + """This method blocks until it obtained the lock, or raises :class:`IOError` if it ran out of time or if the parent directory was not available anymore. If this method returns, you are guaranteed to own the lock. @@ -1105,23 +1128,32 @@ def _obtain_lock(self) -> None: class IterableList(List[T_IterableObj]): - """ - List of iterable objects allowing to query an object by id or by named index:: + """List of iterable objects allowing to query an object by id or by named index:: heads = repo.heads heads.master heads['master'] heads[0] - Iterable parent objects = [Commit, SubModule, Reference, FetchInfo, PushInfo] - Iterable via inheritance = [Head, TagReference, RemoteReference] + Iterable parent objects: + + * :class:`Commit ` + * :class:`Submodule ` + * :class:`Reference ` + * :class:`FetchInfo ` + * :class:`PushInfo ` + + Iterable via inheritance: - It requires an id_attribute name to be set which will be queried from its + * :class:`Head ` + * :class:`TagReference ` + * :class:`RemoteReference ` + + This requires an ``id_attribute`` name to be set which will be queried from its contained items to have a means for comparison. - A prefix can be specified which is to be used in case the id returned by the - items always contains a prefix that does not matter to the user, so it - can be left out. + A prefix can be specified which is to be used in case the id returned by the items + always contains a prefix that does not matter to the user, so it can be left out. """ __slots__ = ("_id_attr", "_prefix") @@ -1198,7 +1230,14 @@ class IterableObj(Protocol): """Defines an interface for iterable items, so there is a uniform way to retrieve and iterate items within the git repository. - Subclasses = [Submodule, Commit, Reference, PushInfo, FetchInfo, Remote] + Subclasses: + + * :class:`Submodule ` + * :class:`Commit ` + * :class:`Reference ` + * :class:`PushInfo ` + * :class:`FetchInfo ` + * :class:`Remote ` """ __slots__ = () @@ -1211,11 +1250,12 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator[T_Itera # Return-typed to be compatible with subtypes e.g. Remote. """Find (all) items of this type. - Subclasses can specify ``args`` and ``kwargs`` differently, and may use them for + Subclasses can specify `args` and `kwargs` differently, and may use them for filtering. However, when the method is called with no additional positional or keyword arguments, subclasses are obliged to to yield all items. - :return: Iterator yielding Items + :return: + :class:`~collections.abc.Iterator` yielding Items """ raise NotImplementedError("To be implemented by Subclass") @@ -1225,11 +1265,13 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> IterableList[T_I For more information about the arguments, see :meth:`iter_items`. - :note: Favor the :meth:`iter_items` method as it will avoid eagerly collecting - all items. When there are many items, that can slow performance and increase + :note: + Favor the :meth:`iter_items` method as it will avoid eagerly collecting all + items. When there are many items, that can slow performance and increase memory usage. - :return: list(Item,...) list of item instances + :return: + list(Item,...) list of item instances """ out_list: IterableList = IterableList(cls._id_attribute_) out_list.extend(cls.iter_items(repo, *args, **kwargs)) @@ -1272,7 +1314,8 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: See :meth:`IterableObj.iter_items` for details on usage. - :return: Iterator yielding Items + :return: + Iterator yielding Items """ raise NotImplementedError("To be implemented by Subclass") @@ -1284,7 +1327,8 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Any: See :meth:`IterableObj.list_items` for details on usage. - :return: list(Item,...) list of item instances + :return: + list(Item,...) list of item instances """ out_list: Any = IterableList(cls._id_attribute_) out_list.extend(cls.iter_items(repo, *args, **kwargs)) From 29c63ac8d7c75e4863cd355610da3cabd48a421c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 24 Feb 2024 21:16:49 -0500 Subject: [PATCH 0671/1392] Format first Git.execute overload stub like the others This makes it easier to read along with, and compare to, the other overloads. --- git/cmd.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index eb6c235c7..d3aa5f544 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -922,7 +922,12 @@ def version_info(self) -> Tuple[int, ...]: return self._version_info @overload - def execute(self, command: Union[str, Sequence[Any]], *, as_process: Literal[True]) -> "AutoInterrupt": + def execute( + self, + command: Union[str, Sequence[Any]], + *, + as_process: Literal[True], + ) -> "AutoInterrupt": ... @overload From cd8a3123e322ea6166b9970a87f7cc2e892d2316 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 24 Feb 2024 21:21:54 -0500 Subject: [PATCH 0672/1392] Show full-path refresh() in failure message differently This presents it more symbolically, rather than in an example-like style that could confuse. See discussion in #1844. --- git/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index d3aa5f544..760a9ff12 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -458,7 +458,7 @@ def refresh(cls, path: Union[None, PathLike] = None) -> bool: The git executable must be specified in one of the following ways: - be included in your $PATH - be set via $%s - - explicitly set via git.refresh("/full/path/to/git") + - explicitly set via git.refresh() """ ) % cls._git_exec_env_var From 8ec7e32032e0877144517bd15c0e9b6cdd283667 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 26 Feb 2024 14:21:50 -0500 Subject: [PATCH 0673/1392] Revise docstrings within git.index Along the same lines as e08066c and 1cd73ba. --- git/index/base.py | 531 ++++++++++++++++++++++++++-------------------- git/index/fun.py | 102 +++++---- git/index/typ.py | 13 +- git/index/util.py | 12 +- 4 files changed, 378 insertions(+), 280 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 9d1db4e35..31186b203 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -101,10 +101,12 @@ def _named_temporary_file_for_subprocess(directory: PathLike) -> Generator[str, None, None]: """Create a named temporary file git subprocesses can open, deleting it afterward. - :param directory: The directory in which the file is created. + :param directory: + The directory in which the file is created. - :return: A context manager object that creates the file and provides its name on - entry, and deletes it on exit. + :return: + A context manager object that creates the file and provides its name on entry, + and deletes it on exit. """ if os.name == "nt": fd, name = tempfile.mkstemp(dir=directory) @@ -123,21 +125,21 @@ class IndexFile(LazyMixin, git_diff.Diffable, Serializable): git command function calls wherever possible. This provides custom merging facilities allowing to merge without actually changing - your index or your working tree. This way you can perform own test-merges based - on the index only without having to deal with the working copy. This is useful - in case of partial working trees. + your index or your working tree. This way you can perform own test-merges based on + the index only without having to deal with the working copy. This is useful in case + of partial working trees. - ``Entries`` + Entries: - The index contains an entries dict whose keys are tuples of type IndexEntry - to facilitate access. + The index contains an entries dict whose keys are tuples of type + :class:`~git.index.typ.IndexEntry` to facilitate access. - You may read the entries dict or manipulate it using IndexEntry instance, i.e.:: + You may read the entries dict or manipulate it using IndexEntry instance, i.e.:: - index.entries[index.entry_key(index_entry_instance)] = index_entry_instance + index.entries[index.entry_key(index_entry_instance)] = index_entry_instance - Make sure you use index.write() once you are done manipulating the index directly - before operating on it using the git command. + Make sure you use :meth:`index.write() ` once you are done manipulating the + index directly before operating on it using the git command. """ __slots__ = ("repo", "version", "entries", "_extension_data", "_file_path") @@ -149,9 +151,9 @@ class IndexFile(LazyMixin, git_diff.Diffable, Serializable): """Flags for a submodule.""" def __init__(self, repo: "Repo", file_path: Union[PathLike, None] = None) -> None: - """Initialize this Index instance, optionally from the given ``file_path``. + """Initialize this Index instance, optionally from the given `file_path`. - If no file_path is given, we will be created from the current index file. + If no `file_path` is given, we will be created from the current index file. If a stream is not given, the stream will be initialized from the current repository's index on demand. @@ -230,25 +232,22 @@ def write( """Write the current state to our file path or to the given one. :param file_path: - If None, we will write to our stored file path from which we have - been initialized. Otherwise we write to the given file path. - Please note that this will change the file_path of this index to - the one you gave. + If None, we will write to our stored file path from which we have been + initialized. Otherwise we write to the given file path. Please note that + this will change the file_path of this index to the one you gave. :param ignore_extension_data: - If True, the TREE type extension data read in the index will not - be written to disk. NOTE that no extension data is actually written. - Use this if you have altered the index and - would like to use git-write-tree afterwards to create a tree - representing your written changes. - If this data is present in the written index, git-write-tree - will instead write the stored/cached tree. - Alternatively, use IndexFile.write_tree() to handle this case - automatically. + If True, the TREE type extension data read in the index will not be written + to disk. NOTE that no extension data is actually written. + Use this if you have altered the index and would like to use git-write-tree + afterwards to create a tree representing your written changes. + If this data is present in the written index, git-write-tree will instead + write the stored/cached tree. + Alternatively, use :meth:`write_tree` to handle this case automatically. """ # Make sure we have our entries read before getting a write lock. - # Otherwise it would be done when streaming. This can happen if one - # doesn't change the index, but writes it right away. + # Otherwise it would be done when streaming. + # This can happen if one doesn't change the index, but writes it right away. self.entries lfd = LockedFD(file_path or self._file_path) stream = lfd.open(write=True, stream=True) @@ -268,17 +267,17 @@ def write( @post_clear_cache @default_index def merge_tree(self, rhs: Treeish, base: Union[None, Treeish] = None) -> "IndexFile": - """Merge the given rhs treeish into the current index, possibly taking + """Merge the given `rhs` treeish into the current index, possibly taking a common base treeish into account. - As opposed to the :func:`IndexFile.from_tree` method, this allows you to use an - already existing tree as the left side of the merge. + As opposed to the :func:`from_tree` method, this allows you to use an already + existing tree as the left side of the merge. :param rhs: Treeish reference pointing to the 'other' side of the merge. :param base: - Optional treeish reference pointing to the common base of 'rhs' and this + Optional treeish reference pointing to the common base of `rhs` and this index which equals lhs. :return: @@ -289,7 +288,7 @@ def merge_tree(self, rhs: Treeish, base: Union[None, Treeish] = None) -> "IndexF If there is a merge conflict. The error will be raised at the first conflicting path. If you want to have proper merge resolution to be done by yourself, you have to commit the changed index (or make a valid tree from - it) and retry with a three-way index.from_tree call. + it) and retry with a three-way :meth:`index.from_tree ` call. """ # -i : ignore working tree status # --aggressive : handle more merge cases @@ -308,15 +307,16 @@ def new(cls, repo: "Repo", *tree_sha: Union[str, Tree]) -> "IndexFile": This method behaves like ``git-read-tree --aggressive`` when doing the merge. - :param repo: The repository treeish are located in. + :param repo: + The repository treeish are located in. :param tree_sha: 20 byte or 40 byte tree sha or tree objects. :return: - New IndexFile instance. Its path will be undefined. - If you intend to write such a merged Index, supply an alternate file_path - to its 'write' method. + New :class:`IndexFile` instance. Its path will be undefined. + If you intend to write such a merged Index, supply an alternate + ``file_path`` to its :meth:`write` method. """ tree_sha_bytes: List[bytes] = [to_bin_sha(str(t)) for t in tree_sha] base_entries = aggressive_tree_merge(repo.odb, tree_sha_bytes) @@ -335,37 +335,40 @@ def new(cls, repo: "Repo", *tree_sha: Union[str, Tree]) -> "IndexFile": @classmethod def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile": - """Merge the given treeish revisions into a new index which is returned. + R"""Merge the given treeish revisions into a new index which is returned. The original index will remain unaltered. :param repo: The repository treeish are located in. :param treeish: - One, two or three Tree Objects, Commits or 40 byte hexshas. The result - changes according to the amount of trees. - If 1 Tree is given, it will just be read into a new index - If 2 Trees are given, they will be merged into a new index using a - two way merge algorithm. Tree 1 is the 'current' tree, tree 2 is the 'other' - one. It behaves like a fast-forward. - If 3 Trees are given, a 3-way merge will be performed with the first tree - being the common ancestor of tree 2 and tree 3. Tree 2 is the 'current' tree, - tree 3 is the 'other' one. + One, two or three :class:`~git.objects.tree.Tree` objects, + :class:`~git.objects.commit.Commit`\s or 40 byte hexshas. + + The result changes according to the amount of trees: + + 1. If 1 Tree is given, it will just be read into a new index. + 2. If 2 Trees are given, they will be merged into a new index using a two + way merge algorithm. Tree 1 is the 'current' tree, tree 2 is the 'other' + one. It behaves like a fast-forward. + 3. If 3 Trees are given, a 3-way merge will be performed with the first tree + being the common ancestor of tree 2 and tree 3. Tree 2 is the 'current' + tree, tree 3 is the 'other' one. :param kwargs: - Additional arguments passed to git-read-tree. + Additional arguments passed to ``git read-tree``. :return: - New IndexFile instance. It will point to a temporary index location which - does not exist anymore. If you intend to write such a merged Index, supply - an alternate file_path to its 'write' method. + New :class:`IndexFile` instance. It will point to a temporary index location + which does not exist anymore. If you intend to write such a merged Index, + supply an alternate ``file_path`` to its :meth:`write` method. :note: - In the three-way merge case, --aggressive will be specified to automatically - resolve more cases in a commonly correct manner. Specify trivial=True as kwarg - to override that. + In the three-way merge case, ``--aggressive`` will be specified to + automatically resolve more cases in a commonly correct manner. Specify + ``trivial=True`` as a keyword argument to override that. - As the underlying git-read-tree command takes into account the current + As the underlying ``git read-tree`` command takes into account the current index, it will be temporarily moved out of the way to prevent any unexpected interference. """ @@ -387,8 +390,8 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile arg_list.append("--index-output=%s" % tmp_index) arg_list.extend(treeish) - # Move the current index out of the way - otherwise the merge may fail - # as it considers existing entries. Moving it essentially clears the index. + # Move the current index out of the way - otherwise the merge may fail as it + # considers existing entries. Moving it essentially clears the index. # Unfortunately there is no 'soft' way to do it. # The TemporaryFileSwap ensures the original file gets put back. with TemporaryFileSwap(join_path_native(repo.git_dir, "index")): @@ -402,12 +405,13 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile @unbare_repo def _iter_expand_paths(self: "IndexFile", paths: Sequence[PathLike]) -> Iterator[PathLike]: - """Expand the directories in list of paths to the corresponding paths accordingly. + """Expand the directories in list of paths to the corresponding paths + accordingly. :note: - git will add items multiple times even if a glob overlapped - with manually specified paths or if paths where specified multiple - times - we respect that and do not prune. + git will add items multiple times even if a glob overlapped with manually + specified paths or if paths where specified multiple times - we respect that + and do not prune. """ def raise_exc(e: Exception) -> NoReturn: @@ -436,11 +440,11 @@ def raise_exc(e: Exception) -> NoReturn: if not os.path.exists(abs_path) and ("?" in abs_path or "*" in abs_path or "[" in abs_path): resolved_paths = glob.glob(abs_path) # not abs_path in resolved_paths: - # a glob() resolving to the same path we are feeding it with - # is a glob() that failed to resolve. If we continued calling - # ourselves we'd endlessly recurse. If the condition below - # evaluates to true then we are likely dealing with a file - # whose name contains wildcard characters. + # A glob() resolving to the same path we are feeding it with is a + # glob() that failed to resolve. If we continued calling ourselves + # we'd endlessly recurse. If the condition below evaluates to true + # then we are likely dealing with a file whose name contains wildcard + # characters. if abs_path not in resolved_paths: for f in self._iter_expand_paths(glob.glob(abs_path)): yield str(f).replace(rs, "") @@ -468,22 +472,27 @@ def _write_path_to_stdin( fprogress: Callable[[PathLike, bool, PathLike], None], read_from_stdout: bool = True, ) -> Union[None, str]: - """Write path to proc.stdin and make sure it processes the item, including progress. + """Write path to ``proc.stdin`` and make sure it processes the item, including + progress. - :return: stdout string + :return: + stdout string - :param read_from_stdout: if True, proc.stdout will be read after the item - was sent to stdin. In that case, it will return None. + :param read_from_stdout: + If True, proc.stdout will be read after the item was sent to stdin. In that + case, it will return None. - :note: There is a bug in git-update-index that prevents it from sending - reports just in time. This is why we have a version that tries to - read stdout and one which doesn't. In fact, the stdout is not - important as the piped-in files are processed anyway and just in time. + :note: + There is a bug in git-update-index that prevents it from sending reports + just in time. This is why we have a version that tries to read stdout and + one which doesn't. In fact, the stdout is not important as the piped-in + files are processed anyway and just in time. - :note: Newlines are essential here, gits behaviour is somewhat inconsistent - on this depending on the version, hence we try our best to deal with - newlines carefully. Usually the last newline will not be sent, instead - we will close stdin to break the pipe. + :note: + Newlines are essential here, git's behaviour is somewhat inconsistent on + this depending on the version, hence we try our best to deal with newlines + carefully. Usually the last newline will not be sent, instead we will close + stdin to break the pipe. """ fprogress(filepath, False, item) rval: Union[None, str] = None @@ -506,12 +515,13 @@ def iter_blobs( self, predicate: Callable[[Tuple[StageType, Blob]], bool] = lambda t: True ) -> Iterator[Tuple[StageType, Blob]]: """ - :return: Iterator yielding tuples of Blob objects and stages, tuple(stage, Blob) + :return: + Iterator yielding tuples of Blob objects and stages, tuple(stage, Blob). :param predicate: Function(t) returning True if tuple(stage, Blob) should be yielded by the - iterator. A default filter, the BlobFilter, allows you to yield blobs - only if they match a given list of paths. + iterator. A default filter, the `~git.index.typ.BlobFilter`, allows you to + yield blobs only if they match a given list of paths. """ for entry in self.entries.values(): blob = entry.to_blob(self.repo) @@ -524,14 +534,13 @@ def iter_blobs( def unmerged_blobs(self) -> Dict[PathLike, List[Tuple[StageType, Blob]]]: """ :return: - Dict(path : list( tuple( stage, Blob, ...))), being - a dictionary associating a path in the index with a list containing - sorted stage/blob pairs. + Dict(path : list( tuple( stage, Blob, ...))), being a dictionary associating + a path in the index with a list containing sorted stage/blob pairs. :note: - Blobs that have been removed in one side simply do not exist in the - given stage. I.e. a file removed on the 'other' branch whose entries - are at stage 3 will not have a stage 3 entry. + Blobs that have been removed in one side simply do not exist in the given + stage. That is, a file removed on the 'other' branch whose entries are at + stage 3 will not have a stage 3 entry. """ is_unmerged_blob = lambda t: t[0] != 0 path_map: Dict[PathLike, List[Tuple[StageType, Blob]]] = {} @@ -556,9 +565,11 @@ def resolve_blobs(self, iter_blobs: Iterator[Blob]) -> "IndexFile": For each path there may only be one blob, otherwise a ValueError will be raised claiming the path is already at stage 0. - :raise ValueError: if one of the blobs already existed at stage 0 + :raise ValueError: + If one of the blobs already existed at stage 0. - :return: self + :return: + self :note: You will have to write the index manually once you are done, i.e. @@ -588,8 +599,9 @@ def update(self) -> "IndexFile": """Reread the contents of our index file, discarding all cached information we might have. - :note: This is a possibly dangerous operations as it will discard your changes - to index.entries. + :note: + This is a possibly dangerous operations as it will discard your changes to + :attr:`index.entries `. :return: self """ @@ -598,16 +610,19 @@ def update(self) -> "IndexFile": return self def write_tree(self) -> Tree: - """Write this index to a corresponding Tree object into the repository's - object database and return it. + """Write this index to a corresponding :class:`~git.objects.tree.Tree` object + into the repository's object database and return it. - :return: Tree object representing this index. + :return: + :class:`~git.objects.tree.Tree` object representing this index. - :note: The tree will be written even if one or more objects the tree refers to - does not yet exist in the object database. This could happen if you added - Entries to the index directly. + :note: + The tree will be written even if one or more objects the tree refers to does + not yet exist in the object database. This could happen if you added entries + to the index directly. - :raise ValueError: if there are no entries in the cache + :raise ValueError: + If there are no entries in the cache. :raise UnmergedEntriesError: """ @@ -620,8 +635,8 @@ def write_tree(self) -> Tree: # Copy changed trees only. mdb.stream_copy(mdb.sha_iter(), self.repo.odb) - # Note: Additional deserialization could be saved if write_tree_from_cache - # would return sorted tree entries. + # Note: Additional deserialization could be saved if write_tree_from_cache would + # return sorted tree entries. root_tree = Tree(self.repo, binsha, path="") root_tree._cache = tree_items return root_tree @@ -639,8 +654,11 @@ def _process_diff_args( def _to_relative_path(self, path: PathLike) -> PathLike: """ - :return: Version of path relative to our git directory or raise ValueError - if it is not within our git directory""" + :return: Version of path relative to our git directory or raise + :class:`ValueError` if it is not within our git directory. + + :raise ValueError: + """ if not osp.isabs(path): return path if self.repo.bare: @@ -672,8 +690,12 @@ def _preprocess_add_items( return paths, entries def _store_path(self, filepath: PathLike, fprogress: Callable) -> BaseIndexEntry: - """Store file at filepath in the database and return the base index entry - Needs the git_working_dir decorator active ! This must be assured in the calling code""" + """Store file at filepath in the database and return the base index entry. + + :note: + This needs the git_working_dir decorator active! + This must be ensured in the calling code. + """ st = os.lstat(filepath) # Handles non-symlinks as well. if S_ISLNK(st.st_mode): # In PY3, readlink is a string, but we need bytes. @@ -744,8 +766,8 @@ def add( write: bool = True, write_extension_data: bool = False, ) -> List[BaseIndexEntry]: - """Add files from the working tree, specific blobs or BaseIndexEntries - to the index. + R"""Add files from the working tree, specific blobs, or + :class:`~git.index.typ.BaseIndexEntry`\s to the index. :param items: Multiple types of items are supported, types can be mixed within one call. @@ -753,9 +775,10 @@ def add( relative or absolute. - path string + Strings denote a relative or absolute path into the repository pointing - to an existing file, e.g., CHANGES, lib/myfile.ext, - '/home/gitrepo/lib/myfile.ext'. + to an existing file, e.g., ``CHANGES``, `lib/myfile.ext``, + ``/home/gitrepo/lib/myfile.ext``. Absolute paths must start with working tree directory of this index's repository to be considered valid. For example, if it was initialized @@ -769,11 +792,12 @@ def add( directories like ``lib``, which will add all the files within the directory and subdirectories. - This equals a straight git-add. + This equals a straight ``git add``. They are added at stage 0. - - Blob or Submodule object + - :class:~`git.objects.blob.Blob` or :class:`~git.objects.submodule.base.Submodule` object + Blobs are added as they are assuming a valid mode is set. The file they refer to may or may not exist in the file system, but must @@ -787,53 +811,57 @@ def add( except that the mode you have set will be kept. This allows you to create symlinks by settings the mode respectively and writing the target of the symlink directly into the file. This equals a default - Linux-Symlink which is not dereferenced automatically, except that it + Linux symlink which is not dereferenced automatically, except that it can be created on filesystems not supporting it as well. - Please note that globs or directories are not allowed in Blob objects. + Please note that globs or directories are not allowed in + :class:~`git.objects.blob.Blob` objects. They are added at stage 0. - - BaseIndexEntry or type + - :class:`~git.index.typ.BaseIndexEntry` or type + Handling equals the one of Blob objects, but the stage may be explicitly set. Please note that Index Entries require binary sha's. :param force: **CURRENTLY INEFFECTIVE** If True, otherwise ignored or excluded files will be added anyway. - As opposed to the git-add command, we enable this flag by default - as the API user usually wants the item to be added even though - they might be excluded. + As opposed to the ``git add`` command, we enable this flag by default as the + API user usually wants the item to be added even though they might be + excluded. :param fprogress: - Function with signature f(path, done=False, item=item) called for each - path to be added, one time once it is about to be added where done==False - and once after it was added where done=True. - item is set to the actual item we handle, either a Path or a BaseIndexEntry - Please note that the processed path is not guaranteed to be present - in the index already as the index is currently being processed. + Function with signature ``f(path, done=False, item=item)`` called for each + path to be added, one time once it is about to be added where ``done=False`` + and once after it was added where ``done=True``. + ``item`` is set to the actual item we handle, either a Path or a + :class:`~git.index.typ.BaseIndexEntry`. + Please note that the processed path is not guaranteed to be present in the + index already as the index is currently being processed. :param path_rewriter: - Function with signature (string) func(BaseIndexEntry) function returning a - path for each passed entry which is the path to be actually recorded for the - object created from entry.path. This allows you to write an index which is - not identical to the layout of the actual files on your hard-disk. If not - None and ``items`` contain plain paths, these paths will be converted to - Entries beforehand and passed to the path_rewriter. Please note that - entry.path is relative to the git repository. + Function, with signature ``(string) func(BaseIndexEntry)``, returning a path + for each passed entry which is the path to be actually recorded for the + object created from :attr:`entry.path `. + This allows you to write an index which is not identical to the layout of + the actual files on your hard-disk. If not None and `items` contain plain + paths, these paths will be converted to Entries beforehand and passed to the + path_rewriter. Please note that ``entry.path`` is relative to the git + repository. :param write: - If True, the index will be written once it was altered. Otherwise - the changes only exist in memory and are not available to git commands. + If True, the index will be written once it was altered. Otherwise the + changes only exist in memory and are not available to git commands. :param write_extension_data: If True, extension data will be written back to the index. This can lead to issues in case it is containing the 'TREE' extension, which will cause the - `git commit` command to write an old tree, instead of a new one representing - the now changed index. + ``git commit`` command to write an old tree, instead of a new one + representing the now changed index. This doesn't matter if you use :meth:`IndexFile.commit`, which ignores the - `TREE` extension altogether. You should set it to True if you intend to use + 'TREE' extension altogether. You should set it to True if you intend to use :meth:`IndexFile.commit` exclusively while maintaining support for third-party extensions. Besides that, you can usually safely ignore the built-in extensions when using GitPython on repositories that are not @@ -843,18 +871,20 @@ def add( http://opensource.apple.com/source/Git/Git-26/src/git-htmldocs/technical/index-format.txt :return: - List(BaseIndexEntries) representing the entries just actually added. + List of :class:`~git.index.typ.BaseIndexEntry`\s representing the entries + just actually added. :raise OSError: - If a supplied Path did not exist. Please note that BaseIndexEntry - Objects that do not have a null sha will be added even if their paths - do not exist. + If a supplied Path did not exist. Please note that + :class:`~git.index.typ.BaseIndexEntry` objects that do not have a null sha + will be added even if their paths do not exist. """ - # Sort the entries into strings and Entries. Blobs are converted to entries automatically. + # Sort the entries into strings and Entries. + # Blobs are converted to entries automatically. # Paths can be git-added. For everything else we use git-update-index. paths, entries = self._preprocess_add_items(items) entries_added: List[BaseIndexEntry] = [] - # This code needs a working tree, therefore we try not to run it unless required. + # This code needs a working tree, so we try not to run it unless required. # That way, we are OK on a bare repository as well. # If there are no paths, the rewriter has nothing to do either. if paths: @@ -897,7 +927,8 @@ def handle_null_entries(self: "IndexFile") -> None: # END null_entry handling # REWRITE PATHS - # If we have to rewrite the entries, do so now, after we have generated all object sha's. + # If we have to rewrite the entries, do so now, after we have generated all + # object sha's. if path_rewriter: for i, e in enumerate(entries): entries[i] = BaseIndexEntry((e.mode, e.binsha, e.stage, path_rewriter(e))) @@ -955,26 +986,28 @@ def remove( working_tree: bool = False, **kwargs: Any, ) -> List[str]: - """Remove the given items from the index and optionally from - the working tree as well. + R"""Remove the given items from the index and optionally from the working tree + as well. :param items: Multiple types of items are supported which may be be freely mixed. - path string + Remove the given path at all stages. If it is a directory, you must - specify the r=True keyword argument to remove all file entries - below it. If absolute paths are given, they will be converted - to a path relative to the git repository directory containing - the working tree + specify the ``r=True`` keyword argument to remove all file entries below + it. If absolute paths are given, they will be converted to a path + relative to the git repository directory containing the working tree + + The path string may include globs, such as \*.c. - The path string may include globs, such as \\*.c. + - :class:~`git.objects.blob.Blob` object - - Blob Object Only the path portion is used in this case. - - BaseIndexEntry or compatible type - The only relevant information here Yis the path. The stage is ignored. + - :class:`~git.index.typ.BaseIndexEntry` or compatible type + + The only relevant information here is the path. The stage is ignored. :param working_tree: If True, the entry will also be removed from the working tree, physically @@ -982,14 +1015,15 @@ def remove( in it. :param kwargs: - Additional keyword arguments to be passed to git-rm, such - as 'r' to allow recursive removal of + Additional keyword arguments to be passed to ``git rm``, such as ``r`` to + allow recursive removal. :return: - List(path_string, ...) list of repository relative paths that have - been removed effectively. - This is interesting to know in case you have provided a directory or - globs. Paths are relative to the repository. + List(path_string, ...) list of repository relative paths that have been + removed effectively. + + This is interesting to know in case you have provided a directory or globs. + Paths are relative to the repository. """ args = [] if not working_tree: @@ -1013,28 +1047,38 @@ def move( **kwargs: Any, ) -> List[Tuple[str, str]]: """Rename/move the items, whereas the last item is considered the destination of - the move operation. If the destination is a file, the first item (of two) - must be a file as well. If the destination is a directory, it may be preceded - by one or more directories or files. + the move operation. + + If the destination is a file, the first item (of two) must be a file as well. + + If the destination is a directory, it may be preceded by one or more directories + or files. The working tree will be affected in non-bare repositories. - :parma items: + :param items: Multiple types of items are supported, please see the :meth:`remove` method for reference. + :param skip_errors: - If True, errors such as ones resulting from missing source files will - be skipped. + If True, errors such as ones resulting from missing source files will be + skipped. + :param kwargs: - Additional arguments you would like to pass to git-mv, such as dry_run - or force. + Additional arguments you would like to pass to ``git mv``, such as + ``dry_run`` or ``force``. - :return: List(tuple(source_path_string, destination_path_string), ...) - A list of pairs, containing the source file moved as well as its - actual destination. Relative to the repository root. + :return: + List(tuple(source_path_string, destination_path_string), ...) + + A list of pairs, containing the source file moved as well as its actual + destination. Relative to the repository root. + + :raise ValueError: + If only one item was given. - :raise ValueError: If only one item was given - :raise GitCommandError: If git could not handle your request + :raise GitCommandError: + If git could not handle your request. """ args = [] if skip_errors: @@ -1047,13 +1091,13 @@ def move( was_dry_run = kwargs.pop("dry_run", kwargs.pop("n", None)) kwargs["dry_run"] = True - # First execute rename in dryrun so the command tells us what it actually does + # First execute rename in dry run so the command tells us what it actually does # (for later output). out = [] mvlines = self.repo.git.mv(args, paths, **kwargs).splitlines() - # Parse result - first 0:n/2 lines are 'checking ', the remaining ones - # are the 'renaming' ones which we parse. + # Parse result - first 0:n/2 lines are 'checking ', the remaining ones are the + # 'renaming' ones which we parse. for ln in range(int(len(mvlines) / 2), len(mvlines)): tokens = mvlines[ln].split(" to ") assert len(tokens) == 2, "Too many tokens in %s" % mvlines[ln] @@ -1066,7 +1110,7 @@ def move( # Either prepare for the real run, or output the dry-run result. if was_dry_run: return out - # END handle dryrun + # END handle dry run # Now apply the actual operation. kwargs.pop("dry_run") @@ -1085,17 +1129,22 @@ def commit( commit_date: Union[datetime.datetime, str, None] = None, skip_hooks: bool = False, ) -> Commit: - """Commit the current default index file, creating a Commit object. + """Commit the current default index file, creating a + :class:`~git.objects.commit.Commit` object. For more information on the arguments, see :meth:`Commit.create_from_tree `. - :note: If you have manually altered the :attr:`entries` member of this instance, - don't forget to :meth:`write` your changes to disk beforehand. - Passing ``skip_hooks=True`` is the equivalent of using ``-n`` - or ``--no-verify`` on the command line. + :note: + If you have manually altered the :attr:`entries` member of this instance, + don't forget to :meth:`write` your changes to disk beforehand. + + :note: + Passing ``skip_hooks=True`` is the equivalent of using ``-n`` or + ``--no-verify`` on the command line. - :return: :class:`Commit` object representing the new commit + :return: + :class:`~git.objects.commit.Commit` object representing the new commit """ if not skip_hooks: run_commit_hook("pre-commit", self) @@ -1160,44 +1209,49 @@ def checkout( """Check out the given paths or all files from the version known to the index into the working tree. - :note: Be sure you have written pending changes using the :meth:`write` method - in case you have altered the entries dictionary directly. + :note: + Be sure you have written pending changes using the :meth:`write` method in + case you have altered the entries dictionary directly. :param paths: If None, all paths in the index will be checked out. Otherwise an iterable - of relative or absolute paths or a single path pointing to files or directories - in the index is expected. + of relative or absolute paths or a single path pointing to files or + directories in the index is expected. :param force: - If True, existing files will be overwritten even if they contain local modifications. - If False, these will trigger a :class:`CheckoutError`. + If True, existing files will be overwritten even if they contain local + modifications. + If False, these will trigger a :class:`~git.exc.CheckoutError`. :param fprogress: - see :func:`IndexFile.add` for signature and explanation. + See :meth:`IndexFile.add` for signature and explanation. + The provided progress information will contain None as path and item if no - explicit paths are given. Otherwise progress information will be send - prior and after a file has been checked out. + explicit paths are given. Otherwise progress information will be send prior + and after a file has been checked out. :param kwargs: - Additional arguments to be passed to git-checkout-index. + Additional arguments to be passed to ``git checkout-index``. :return: - iterable yielding paths to files which have been checked out and are + Iterable yielding paths to files which have been checked out and are guaranteed to match the version stored in the index. - :raise exc.CheckoutError: - If at least one file failed to be checked out. This is a summary, - hence it will checkout as many files as it can anyway. - If one of files or directories do not exist in the index - ( as opposed to the original git command who ignores them ). - Raise GitCommandError if error lines could not be parsed - this truly is - an exceptional state. - - .. note:: The checkout is limited to checking out the files in the - index. Files which are not in the index anymore and exist in - the working tree will not be deleted. This behaviour is fundamentally - different to *head.checkout*, i.e. if you want git-checkout like behaviour, - use head.checkout instead of index.checkout. + :raise git.exc.CheckoutError: + If at least one file failed to be checked out. This is a summary, hence it + will checkout as many files as it can anyway. + If one of files or directories do not exist in the index (as opposed to the + original git command, which ignores them). + + :raise git.exc.GitCommandError: + If error lines could not be parsed - this truly is an exceptional state. + + :note: + The checkout is limited to checking out the files in the index. Files which + are not in the index anymore and exist in the working tree will not be + deleted. This behaviour is fundamentally different to ``head.checkout``, + i.e. if you want ``git checkout`` like behaviour, use ``head.checkout`` + instead of ``index.checkout``. """ args = ["--index"] if force: @@ -1276,9 +1330,9 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik if isinstance(paths, str): paths = [paths] - # Make sure we have our entries loaded before we start checkout_index, - # which will hold a lock on it. We try to get the lock as well during - # our entries initialization. + # Make sure we have our entries loaded before we start checkout_index, which + # will hold a lock on it. We try to get the lock as well during our entries + # initialization. self.entries args.append("--stdin") @@ -1339,40 +1393,50 @@ def reset( head: bool = False, **kwargs: Any, ) -> "IndexFile": - """Reset the index to reflect the tree at the given commit. This will not - adjust our HEAD reference as opposed to HEAD.reset by default. + """Reset the index to reflect the tree at the given commit. This will not adjust + our ``HEAD`` reference by default, as opposed to + :meth:`HEAD.reset `. :param commit: - Revision, Reference or Commit specifying the commit we should represent. - If you want to specify a tree only, use IndexFile.from_tree and overwrite - the default index. + Revision, :class:`~git.refs.reference.Reference` or + :class:`~git.objects.commit.Commit` specifying the commit we should + represent. + + If you want to specify a tree only, use :meth:`IndexFile.from_tree` and + overwrite the default index. :param working_tree: If True, the files in the working tree will reflect the changed index. - If False, the working tree will not be touched + If False, the working tree will not be touched. Please note that changes to the working copy will be discarded without - warning ! + warning! :param head: If True, the head will be set to the given commit. This is False by default, - but if True, this method behaves like HEAD.reset. + but if True, this method behaves like + :meth:`HEAD.reset `. - :param paths: if given as an iterable of absolute or repository-relative paths, - only these will be reset to their state at the given commit-ish. + :param paths: + If given as an iterable of absolute or repository-relative paths, only these + will be reset to their state at the given commit-ish. The paths need to exist at the commit, otherwise an exception will be raised. :param kwargs: - Additional keyword arguments passed to git-reset + Additional keyword arguments passed to ``git reset``. - .. note:: IndexFile.reset, as opposed to HEAD.reset, will not delete any files - in order to maintain a consistent working tree. Instead, it will just - check out the files according to their state in the index. - If you want git-reset like behaviour, use *HEAD.reset* instead. + :note: + :meth:`IndexFile.reset`, as opposed to + :meth:`HEAD.reset `, will not delete any files in + order to maintain a consistent working tree. Instead, it will just check out + the files according to their state in the index. + If you want ``git reset``-like behaviour, use + :meth:`HEAD.reset ` instead. - :return: self""" - # What we actually want to do is to merge the tree into our existing - # index, which is what git-read-tree does. + :return: self + """ + # What we actually want to do is to merge the tree into our existing index, + # which is what git-read-tree does. new_inst = type(self).from_tree(self.repo, commit) if not paths: self.entries = new_inst.entries @@ -1413,14 +1477,15 @@ def diff( create_patch: bool = False, **kwargs: Any, ) -> git_diff.DiffIndex: - """Diff this index against the working copy or a Tree or Commit object. + """Diff this index against the working copy or a :class:`~git.objects.tree.Tree` + or :class:`~git.objects.commit.Commit` object. For documentation of the parameters and return values, see :meth:`Diffable.diff `. :note: - Will only work with indices that represent the default git index as - they have not been initialized with a stream. + Will only work with indices that represent the default git index as they + have not been initialized with a stream. """ # Only run if we are the default repository index. @@ -1430,9 +1495,9 @@ def diff( if other is self.Index: return git_diff.DiffIndex() - # Index against anything but None is a reverse diff with the respective - # item. Handle existing -R flags properly. Transform strings to the object - # so that we can call diff on it. + # Index against anything but None is a reverse diff with the respective item. + # Handle existing -R flags properly. + # Transform strings to the object so that we can call diff on it. if isinstance(other, str): other = self.repo.rev_parse(other) # END object conversion diff --git a/git/index/fun.py b/git/index/fun.py index 580493f6d..785a1c748 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -1,7 +1,8 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Standalone functions to accompany the index implementation and make it more versatile.""" +"""Standalone functions to accompany the index implementation and make it more +versatile.""" from io import BytesIO import os @@ -78,9 +79,15 @@ def _has_file_extension(path: str) -> str: def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: """Run the commit hook of the given name. Silently ignore hooks that do not exist. - :param name: name of hook, like 'pre-commit' - :param index: IndexFile instance - :param args: Arguments passed to hook file + :param name: + Name of hook, like ``pre-commit``. + + :param index: + :class:`~git.index.base.IndexFile` instance. + + :param args: + Arguments passed to hook file. + :raises HookExecutionError: """ hp = hook_path(name, index.repo.git_dir) @@ -121,8 +128,8 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: def stat_mode_to_index_mode(mode: int) -> int: - """Convert the given mode from a stat call to the corresponding index mode - and return it.""" + """Convert the given mode from a stat call to the corresponding index mode and + return it.""" if S_ISLNK(mode): # symlinks return S_IFLNK if S_ISDIR(mode) or S_IFMT(mode) == S_IFGITLINK: # submodules @@ -138,16 +145,19 @@ def write_cache( ) -> None: """Write the cache represented by entries to a stream. - :param entries: **sorted** list of entries + :param entries: + **Sorted** list of entries. - :param stream: stream to wrap into the AdapterStreamCls - it is used for - final output. + :param stream: + Stream to wrap into the AdapterStreamCls - it is used for final output. - :param ShaStreamCls: Type to use when writing to the stream. It produces a sha - while writing to it, before the data is passed on to the wrapped stream + :param ShaStreamCls: + Type to use when writing to the stream. It produces a sha while writing to it, + before the data is passed on to the wrapped stream. - :param extension_data: any kind of data to write as a trailer, it must begin - a 4 byte identifier, followed by its size (4 bytes). + :param extension_data: + Any kind of data to write as a trailer, it must begin a 4 byte identifier, + followed by its size (4 bytes). """ # Wrap the stream into a compatible writer. stream_sha = ShaStreamCls(stream) @@ -197,7 +207,7 @@ def write_cache( def read_header(stream: IO[bytes]) -> Tuple[int, int]: - """Return tuple(version_long, num_entries) from the given stream""" + """Return tuple(version_long, num_entries) from the given stream.""" type_id = stream.read(4) if type_id != b"DIRC": raise AssertionError("Invalid index file header: %r" % type_id) @@ -210,9 +220,13 @@ def read_header(stream: IO[bytes]) -> Tuple[int, int]: def entry_key(*entry: Union[BaseIndexEntry, PathLike, int]) -> Tuple[PathLike, int]: - """:return: Key suitable to be used for the index.entries dictionary + """ + :return: + Key suitable to be used for the :attr:`index.entries + ` dictionary. - :param entry: One instance of type BaseIndexEntry or the path and the stage + :param entry: + One instance of type BaseIndexEntry or the path and the stage. """ # def is_entry_key_tup(entry_key: Tuple) -> TypeGuard[Tuple[PathLike, int]]: @@ -234,12 +248,14 @@ def read_cache( ) -> Tuple[int, Dict[Tuple[PathLike, int], "IndexEntry"], bytes, bytes]: """Read a cache file from the given stream. - :return: tuple(version, entries_dict, extension_data, content_sha) + :return: + tuple(version, entries_dict, extension_data, content_sha) - * version is the integer version number. - * entries dict is a dictionary which maps IndexEntry instances to a path at a stage. - * extension_data is '' or 4 bytes of type + 4 bytes of size + size bytes. - * content_sha is a 20 byte sha on all cache file contents. + * *version* is the integer version number. + * *entries_dict* is a dictionary which maps IndexEntry instances to a path at a + stage. + * *extension_data* is ``""`` or 4 bytes of type + 4 bytes of size + size bytes. + * *content_sha* is a 20 byte sha on all cache file contents. """ version, num_entries = read_header(stream) count = 0 @@ -285,15 +301,25 @@ def read_cache( def write_tree_from_cache( entries: List[IndexEntry], odb: "GitCmdObjectDB", sl: slice, si: int = 0 ) -> Tuple[bytes, List["TreeCacheTup"]]: - """Create a tree from the given sorted list of entries and put the respective + R"""Create a tree from the given sorted list of entries and put the respective trees into the given object database. - :param entries: **Sorted** list of IndexEntries - :param odb: Object database to store the trees in - :param si: Start index at which we should start creating subtrees - :param sl: Slice indicating the range we should process on the entries list - :return: tuple(binsha, list(tree_entry, ...)) a tuple of a sha and a list of - tree entries being a tuple of hexsha, mode, name + :param entries: + **Sorted** list of :class:`~git.index.typ.IndexEntry`\s. + + :param odb: + Object database to store the trees in. + + :param si: + Start index at which we should start creating subtrees. + + :param sl: + Slice indicating the range we should process on the entries list. + + :return: + tuple(binsha, list(tree_entry, ...)) + + A tuple of a sha and a list of tree entries being a tuple of hexsha, mode, name. """ tree_items: List["TreeCacheTup"] = [] @@ -346,15 +372,17 @@ def _tree_entry_to_baseindexentry(tree_entry: "TreeCacheTup", stage: int) -> Bas def aggressive_tree_merge(odb: "GitCmdObjectDB", tree_shas: Sequence[bytes]) -> List[BaseIndexEntry]: - """ - :return: List of BaseIndexEntries representing the aggressive merge of the given - trees. All valid entries are on stage 0, whereas the conflicting ones are left - on stage 1, 2 or 3, whereas stage 1 corresponds to the common ancestor tree, - 2 to our tree and 3 to 'their' tree. - - :param tree_shas: 1, 2 or 3 trees as identified by their binary 20 byte shas. - If 1 or two, the entries will effectively correspond to the last given tree. - If 3 are given, a 3 way merge is performed. + R""" + :return: + List of :class:`~git.index.typ.BaseIndexEntry`\s representing the aggressive + merge of the given trees. All valid entries are on stage 0, whereas the + conflicting ones are left on stage 1, 2 or 3, whereas stage 1 corresponds to the + common ancestor tree, 2 to our tree and 3 to 'their' tree. + + :param tree_shas: + 1, 2 or 3 trees as identified by their binary 20 byte shas. If 1 or two, the + entries will effectively correspond to the last given tree. If 3 are given, a 3 + way merge is performed. """ out: List[BaseIndexEntry] = [] diff --git a/git/index/typ.py b/git/index/typ.py index 894e6f16d..dce67626f 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -58,7 +58,8 @@ def __call__(self, stage_blob: Tuple[StageType, Blob]) -> bool: blob_path: Path = blob_pathlike if isinstance(blob_pathlike, Path) else Path(blob_pathlike) for pathlike in self.paths: path: Path = pathlike if isinstance(pathlike, Path) else Path(pathlike) - # TODO: Change to use `PosixPath.is_relative_to` once Python 3.8 is no longer supported. + # TODO: Change to use `PosixPath.is_relative_to` once Python 3.8 is no + # longer supported. filter_parts: List[str] = path.parts blob_parts: List[str] = blob_path.parts if len(filter_parts) > len(blob_parts): @@ -69,8 +70,11 @@ def __call__(self, stage_blob: Tuple[StageType, Blob]) -> bool: class BaseIndexEntryHelper(NamedTuple): - """Typed namedtuple to provide named attribute access for BaseIndexEntry. - Needed to allow overriding __new__ in child class to preserve backwards compat.""" + """Typed named tuple to provide named attribute access for BaseIndexEntry. + + This is needed to allow overriding ``__new__`` in child class to preserve backwards + compatibility. + """ mode: int binsha: bytes @@ -101,7 +105,8 @@ def __new__( Tuple[int, bytes, int, PathLike, bytes, bytes, int, int, int, int, int], ], ) -> "BaseIndexEntry": - """Override __new__ to allow construction from a tuple for backwards compatibility""" + """Override ``__new__`` to allow construction from a tuple for backwards + compatibility.""" return super().__new__(cls, *inp_tuple) def __str__(self) -> str: diff --git a/git/index/util.py b/git/index/util.py index 125925783..8aedb7284 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -34,8 +34,8 @@ class TemporaryFileSwap: - """Utility class moving a file to a temporary location within the same directory - and moving it back on to where on object deletion.""" + """Utility class moving a file to a temporary location within the same directory and + moving it back on to where on object deletion.""" __slots__ = ("file_path", "tmp_file_path") @@ -66,12 +66,12 @@ def __exit__( def post_clear_cache(func: Callable[..., _T]) -> Callable[..., _T]: """Decorator for functions that alter the index using the git command. This would - invalidate our possibly existing entries dictionary which is why it must be - deleted to allow it to be lazily reread later. + invalidate our possibly existing entries dictionary which is why it must be deleted + to allow it to be lazily reread later. :note: - This decorator will not be required once all functions are implemented - natively which in fact is possible, but probably not feasible performance wise. + This decorator will not be required once all functions are implemented natively + which in fact is possible, but probably not feasible performance wise. """ @wraps(func) From ca2ab6137b2cae27fb5c68dbd63c83bd8e9b429e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 26 Feb 2024 14:25:24 -0500 Subject: [PATCH 0674/1392] Rewrite post_clear_cache note So that it just describes the current situation, rather than suggesting a future direction for IndexFile. This is for #1847. --- git/index/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/index/util.py b/git/index/util.py index 8aedb7284..347e8747f 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -70,8 +70,8 @@ def post_clear_cache(func: Callable[..., _T]) -> Callable[..., _T]: to allow it to be lazily reread later. :note: - This decorator will not be required once all functions are implemented natively - which in fact is possible, but probably not feasible performance wise. + This decorator is required because not all functions related to + :class:`~git.index.base.IndexFile` are implemented natively. """ @wraps(func) From 37421e1b8f0618e1d4e5bd322aeb60d23b58e1b9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 26 Feb 2024 15:06:19 -0500 Subject: [PATCH 0675/1392] Further revise post_clear_cache docstring For #1847. This removes the note, and splits out some related material from the docstring's top line into a second paragraph for readability (since the first sentence of the top line was complete, and described usage). --- git/index/util.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/git/index/util.py b/git/index/util.py index 347e8747f..0bad11571 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -65,13 +65,10 @@ def __exit__( def post_clear_cache(func: Callable[..., _T]) -> Callable[..., _T]: - """Decorator for functions that alter the index using the git command. This would - invalidate our possibly existing entries dictionary which is why it must be deleted - to allow it to be lazily reread later. + """Decorator for functions that alter the index using the git command. - :note: - This decorator is required because not all functions related to - :class:`~git.index.base.IndexFile` are implemented natively. + When a git command alters the index, this invalidates our possibly existing entries + dictionary, which is why it must be deleted to allow it to be lazily reread later. """ @wraps(func) From ca32c226e9d52001e7f2feaa516bb2483210db82 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 26 Feb 2024 15:49:04 -0500 Subject: [PATCH 0676/1392] Condense output_stream description in Git.execute docstring For #1845. --- git/cmd.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 760a9ff12..cab089746 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1018,12 +1018,8 @@ def execute( :param output_stream: If set to a file-like object, data produced by the git command will be - output to the given stream directly. - This feature only has any effect if `as_process` is False. Processes will - always be created with a pipe due to issues with subprocess. - This merely is a workaround as data will be copied from the - output pipe to the given output stream directly. - Judging from the implementation, you shouldn't use this parameter! + copied to the given stream instead of being returned as a string. + This feature only has any effect if `as_process` is False. :param stdout_as_string: If False, the command's standard output will be bytes. Otherwise, it will be From 3813bfbee34640da121209ecea0388b0cd1d39cf Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 26 Feb 2024 17:52:54 -0500 Subject: [PATCH 0677/1392] Clarify Submodule.branch_path documentation This changes "Full (relative) path" to "Complete relative path" in the wording used to describe the branch_path initializer parameter and property of the Submodule class. This is to prevent confusion, since "full path" means something like "absolute path" (and is now used as such in error messages introduced since these docstrings were last edited). --- git/objects/submodule/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index d08e967b8..497e5a06a 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -124,7 +124,7 @@ def __init__( the ``url`` parameter. :param parent_commit: See :meth:`set_parent_commit`. :param url: The URL to the remote repository which is the submodule. - :param branch_path: Full (relative) path to ref to checkout when cloning the + :param branch_path: Complete relative path to ref to checkout when cloning the remote repository. """ super().__init__(repo, binsha, mode, path) @@ -1322,7 +1322,7 @@ def branch(self) -> "Head": @property def branch_path(self) -> PathLike: """ - :return: Full (relative) path as string to the branch we would checkout + :return: Complete relative path as string to the branch we would checkout from the remote and track """ return self._branch_path From 63c62ed801380be88ecdf7d686ddcc7cc13cdb29 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 26 Feb 2024 19:43:59 -0500 Subject: [PATCH 0678/1392] Revise docstrings within git.objects.submodule --- git/objects/submodule/base.py | 82 ++++++++++++------- git/objects/submodule/root.py | 148 +++++++++++++++++++--------------- git/objects/submodule/util.py | 15 ++-- 3 files changed, 147 insertions(+), 98 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 497e5a06a..58b474fdd 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -60,7 +60,8 @@ class UpdateProgress(RemoteProgress): """Class providing detailed progress information to the caller who should - derive from it and implement the ``update(...)`` message.""" + derive from it and implement the + :meth:`update(...) ` message.""" CLONE, FETCH, UPDWKTREE = [1 << x for x in range(RemoteProgress._num_op_codes, RemoteProgress._num_op_codes + 3)] _num_op_codes: int = RemoteProgress._num_op_codes + 3 @@ -76,8 +77,8 @@ class UpdateProgress(RemoteProgress): # IndexObject comes via the util module. It's a 'hacky' fix thanks to Python's import -# mechanism, which causes plenty of trouble if the only reason for packages and -# modules is refactoring - subpackages shouldn't depend on parent packages. +# mechanism, which causes plenty of trouble if the only reason for packages and modules +# is refactoring - subpackages shouldn't depend on parent packages. class Submodule(IndexObject, TraversableIterableObj): """Implements access to a git submodule. They are special in that their sha represents a commit in the submodule's repository which is to be checked out @@ -119,11 +120,19 @@ def __init__( We only document the parameters that differ from :class:`~git.objects.base.IndexObject`. - :param repo: Our parent repository. - :param binsha: Binary sha referring to a commit in the remote repository. See - the ``url`` parameter. - :param parent_commit: See :meth:`set_parent_commit`. - :param url: The URL to the remote repository which is the submodule. + :param repo: + Our parent repository. + + :param binsha: + Binary sha referring to a commit in the remote repository. + See the `url` parameter. + + :param parent_commit: + See :meth:`set_parent_commit`. + + :param url: + The URL to the remote repository which is the submodule. + :param branch_path: Complete relative path to ref to checkout when cloning the remote repository. """ @@ -1313,9 +1322,11 @@ def exists(self) -> bool: @property def branch(self) -> "Head": """ - :return: The branch instance that we are to checkout + :return: + The branch instance that we are to checkout - :raise InvalidGitRepositoryError: If our module is not yet checked out + :raise InvalidGitRepositoryError: + If our module is not yet checked out. """ return mkhead(self.module(), self._branch_path) @@ -1329,7 +1340,10 @@ def branch_path(self) -> PathLike: @property def branch_name(self) -> str: - """:return: The name of the branch, which is the shortest possible branch name""" + """ + :return: + The name of the branch, which is the shortest possible branch name + """ # Use an instance method, for this we create a temporary Head instance # which uses a repository that is available at least (it makes no difference). return git.Head(self.repo, self._branch_path).name @@ -1342,9 +1356,11 @@ def url(self) -> str: @property def parent_commit(self) -> "Commit_ish": """ - :return: Commit instance with the tree containing the .gitmodules file + :return: + Commit instance with the tree containing the ``.gitmodules`` file - :note: Will always point to the current head's commit if it was not set explicitly. + :note: + Will always point to the current head's commit if it was not set explicitly. """ if self._parent_commit is None: return self.repo.commit() @@ -1353,33 +1369,40 @@ def parent_commit(self) -> "Commit_ish": @property def name(self) -> str: """ - :return: The name of this submodule. It is used to identify it within the - .gitmodules file. + :return: + The name of this submodule. It is used to identify it within the + ``.gitmodules`` file. - :note: By default, this is the name is the path at which to find the submodule, - but in GitPython it should be a unique identifier similar to the identifiers + :note: + By default, this is the name is the path at which to find the submodule, but + in GitPython it should be a unique identifier similar to the identifiers used for remotes, which allows to change the path of the submodule easily. """ return self._name def config_reader(self) -> SectionConstraint[SubmoduleConfigParser]: """ - :return: ConfigReader instance which allows you to query the configuration - values of this submodule, as provided by the .gitmodules file. + :return: + ConfigReader instance which allows you to query the configuration values of + this submodule, as provided by the ``.gitmodules`` file. - :note: The config reader will actually read the data directly from the - repository and thus does not need nor care about your working tree. + :note: + The config reader will actually read the data directly from the repository + and thus does not need nor care about your working tree. - :note: Should be cached by the caller and only kept as long as needed. + :note: + Should be cached by the caller and only kept as long as needed. - :raise IOError: If the .gitmodules file/blob could not be read. + :raise IOError: + If the ``.gitmodules`` file/blob could not be read. """ return self._config_parser_constrained(read_only=True) def children(self) -> IterableList["Submodule"]: """ - :return: IterableList(Submodule, ...) an iterable list of submodules instances - which are children of this submodule or 0 if the submodule is not checked out. + :return: + IterableList(Submodule, ...) An iterable list of submodules instances which + are children of this submodule or 0 if the submodule is not checked out. """ return self._get_intermediate_items(self) @@ -1395,7 +1418,10 @@ def iter_items( *Args: Any, **kwargs: Any, ) -> Iterator["Submodule"]: - """:return: Iterator yielding Submodule instances available in the given repository""" + """ + :return: + Iterator yielding Submodule instances available in the given repository + """ try: pc = repo.commit(parent_commit) # Parent commit instance parser = cls._config_parser(repo, pc, read_only=True) @@ -1423,8 +1449,8 @@ def iter_items( entry = index.entries[index.entry_key(p, 0)] sm = Submodule(repo, entry.binsha, entry.mode, entry.path) except KeyError: - # The submodule doesn't exist, probably it wasn't - # removed from the .gitmodules file. + # The submodule doesn't exist, probably it wasn't removed from the + # .gitmodules file. continue # END handle keyerror # END handle critical error diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index dde384bbe..8ef874f90 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -26,7 +26,8 @@ class RootUpdateProgress(UpdateProgress): - """Utility class which adds more opcodes to the UpdateProgress.""" + """Utility class which adds more opcodes to + :class:`~git.objects.submodule.base.UpdateProgress`.""" REMOVE, PATHCHANGE, BRANCHCHANGE, URLCHANGE = [ 1 << x for x in range(UpdateProgress._num_op_codes, UpdateProgress._num_op_codes + 4) @@ -91,43 +92,59 @@ def update( This method behaves smartly by determining changes of the path of a submodules repository, next to changes to the to-be-checked-out commit or the branch to be checked out. This works if the submodules ID does not change. + Additionally it will detect addition and removal of submodules, which will be handled gracefully. - :param previous_commit: If set to a commit-ish, the commit we should use as the - previous commit the HEAD pointed to before it was set to the commit it - points to now. - If None, it defaults to ``HEAD@{1}`` otherwise - :param recursive: if True, the children of submodules will be updated as well - using the same technique. - :param force_remove: If submodules have been deleted, they will be forcibly - removed. Otherwise the update may fail if a submodule's repository cannot be - deleted as changes have been made to it. + :param previous_commit: + If set to a commit-ish, the commit we should use as the previous commit the + HEAD pointed to before it was set to the commit it points to now. + If None, it defaults to ``HEAD@{1}`` otherwise. + + :param recursive: + If True, the children of submodules will be updated as well using the same + technique. + + :param force_remove: + If submodules have been deleted, they will be forcibly removed. Otherwise + the update may fail if a submodule's repository cannot be deleted as changes + have been made to it. (See :meth:`Submodule.update ` for more information.) - :param init: If we encounter a new module which would need to be initialized, - then do it. - :param to_latest_revision: If True, instead of checking out the revision pointed - to by this submodule's sha, the checked out tracking branch will be merged - with the latest remote branch fetched from the repository's origin. + + :param init: + If we encounter a new module which would need to be initialized, then do it. + + :param to_latest_revision: + If True, instead of checking out the revision pointed to by this submodule's + sha, the checked out tracking branch will be merged with the latest remote + branch fetched from the repository's origin. Unless `force_reset` is specified, a local tracking branch will never be reset into its past, therefore the remote branch must be in the future for this to have an effect. - :param force_reset: If True, submodules may checkout or reset their branch even - if the repository has pending changes that would be overwritten, or if the - local tracking branch is in the future of the remote tracking branch and - would be reset into its past. - :param progress: :class:`RootUpdateProgress` instance or None if no progress - should be sent. - :param dry_run: If True, operations will not actually be performed. Progress - messages will change accordingly to indicate the WOULD DO state of the - operation. - :param keep_going: If True, we will ignore but log all errors, and keep going - recursively. Unless `dry_run` is set as well, `keep_going` could cause + + :param force_reset: + If True, submodules may checkout or reset their branch even if the + repository has pending changes that would be overwritten, or if the local + tracking branch is in the future of the remote tracking branch and would be + reset into its past. + + :param progress: + :class:`RootUpdateProgress` instance or None if no progress should be sent. + + :param dry_run: + If True, operations will not actually be performed. Progress messages will + change accordingly to indicate the WOULD DO state of the operation. + + :param keep_going: + If True, we will ignore but log all errors, and keep going recursively. + Unless `dry_run` is set as well, `keep_going` could cause subsequent/inherited errors you wouldn't see otherwise. In conjunction with `dry_run`, this can be useful to anticipate all errors when updating submodules. - :return: self + + :return: + self """ if self.repo.bare: raise InvalidGitRepositoryError("Cannot update submodules in bare repositories") @@ -234,13 +251,14 @@ def update( ################### if sm.url != psm.url: # Add the new remote, remove the old one. - # This way, if the url just changes, the commits will not - # have to be re-retrieved. + # This way, if the url just changes, the commits will not have + # to be re-retrieved. nn = "__new_origin__" smm = sm.module() rmts = smm.remotes - # Don't do anything if we already have the url we search in place. + # Don't do anything if we already have the url we search in + # place. if len([r for r in rmts if r.url == sm.url]) == 0: progress.update( BEGIN | URLCHANGE, @@ -272,17 +290,17 @@ def update( # END if urls match # END for each remote - # If we didn't find a matching remote, but have exactly one, - # we can safely use this one. + # If we didn't find a matching remote, but have exactly + # one, we can safely use this one. if rmt_for_deletion is None: if len(rmts) == 1: rmt_for_deletion = rmts[0] else: - # If we have not found any remote with the original URL - # we may not have a name. This is a special case, - # and its okay to fail here. - # Alternatively we could just generate a unique name and - # leave all existing ones in place. + # If we have not found any remote with the + # original URL we may not have a name. This is a + # special case, and its okay to fail here. + # Alternatively we could just generate a unique + # name and leave all existing ones in place. raise InvalidGitRepositoryError( "Couldn't find original remote-repo at url %r" % psm.url ) @@ -292,19 +310,19 @@ def update( orig_name = rmt_for_deletion.name smm.delete_remote(rmt_for_deletion) # NOTE: Currently we leave tags from the deleted remotes - # as well as separate tracking branches in the possibly totally - # changed repository (someone could have changed the url to - # another project). At some point, one might want to clean - # it up, but the danger is high to remove stuff the user - # has added explicitly. + # as well as separate tracking branches in the possibly + # totally changed repository (someone could have changed + # the url to another project). At some point, one might + # want to clean it up, but the danger is high to remove + # stuff the user has added explicitly. # Rename the new remote back to what it was. smr.rename(orig_name) - # Early on, we verified that the our current tracking branch - # exists in the remote. Now we have to ensure that the - # sha we point to is still contained in the new remote - # tracking branch. + # Early on, we verified that the our current tracking + # branch exists in the remote. Now we have to ensure + # that the sha we point to is still contained in the new + # remote tracking branch. smsha = sm.binsha found = False rref = smr.refs[self.branch_name] @@ -316,10 +334,11 @@ def update( # END for each commit if not found: - # Adjust our internal binsha to use the one of the remote - # this way, it will be checked out in the next step. - # This will change the submodule relative to us, so - # the user will be able to commit the change easily. + # Adjust our internal binsha to use the one of the + # remote this way, it will be checked out in the + # next step. This will change the submodule relative + # to us, so the user will be able to commit the + # change easily. _logger.warning( "Current sha %s was not contained in the tracking\ branch at the new remote, setting it the the remote's tracking branch", @@ -328,7 +347,8 @@ def update( sm.binsha = rref.commit.binsha # END reset binsha - # NOTE: All checkout is performed by the base implementation of update. + # NOTE: All checkout is performed by the base + # implementation of update. # END handle dry_run progress.update( END | URLCHANGE, @@ -342,7 +362,8 @@ def update( # HANDLE PATH CHANGES ##################### if sm.branch_path != psm.branch_path: - # Finally, create a new tracking branch which tracks the new remote branch. + # Finally, create a new tracking branch which tracks the new + # remote branch. progress.update( BEGIN | BRANCHCHANGE, i, @@ -354,8 +375,8 @@ def update( if not dry_run: smm = sm.module() smmr = smm.remotes - # As the branch might not exist yet, we will have to fetch all remotes - # to be sure... + # As the branch might not exist yet, we will have to fetch + # all remotes to be sure... for remote in smmr: remote.fetch(progress=progress) # END for each remote @@ -372,11 +393,12 @@ def update( # END ensure tracking branch exists tbr.set_tracking_branch(find_first_remote_branch(smmr, sm.branch_name)) - # NOTE: All head-resetting is done in the base implementation of update - # but we will have to checkout the new branch here. As it still points - # to the currently checked out commit, we don't do any harm. - # As we don't want to update working-tree or index, changing the ref is - # all there is to do. + # NOTE: All head-resetting is done in the base + # implementation of update but we will have to checkout the + # new branch here. As it still points to the currently + # checked out commit, we don't do any harm. + # As we don't want to update working-tree or index, changing + # the ref is all there is to do. smm.head.reference = tbr # END handle dry_run @@ -409,10 +431,10 @@ def update( keep_going=keep_going, ) - # Update recursively depth first - question is which inconsistent - # state will be better in case it fails somewhere. Defective branch - # or defective depth. The RootSubmodule type will never process itself, - # which was done in the previous expression. + # Update recursively depth first - question is which inconsistent state will + # be better in case it fails somewhere. Defective branch or defective depth. + # The RootSubmodule type will never process itself, which was done in the + # previous expression. if recursive: # The module would exist by now if we are not in dry_run mode. if sm.module_exists(): diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index f8265798d..91e18a26b 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -35,7 +35,7 @@ def sm_section(name: str) -> str: - """:return: Section title used in .gitmodules configuration file""" + """:return: Section title used in ``.gitmodules`` configuration file""" return f'submodule "{name}"' @@ -51,7 +51,8 @@ def mkhead(repo: "Repo", path: PathLike) -> "Head": def find_first_remote_branch(remotes: Sequence["Remote"], branch_name: str) -> "RemoteReference": - """Find the remote branch matching the name of the given branch or raise InvalidGitRepositoryError.""" + """Find the remote branch matching the name of the given branch or raise + :class:`git.exc.InvalidGitRepositoryError`.""" for remote in remotes: try: return remote.refs[branch_name] @@ -69,11 +70,11 @@ def find_first_remote_branch(remotes: Sequence["Remote"], branch_name: str) -> " class SubmoduleConfigParser(GitConfigParser): - """Catches calls to _write, and updates the .gitmodules blob in the index + """Catches calls to _write, and updates the ``.gitmodules`` blob in the index with the new data, if we have written into a stream. - Otherwise it would add the local file to the index to make it correspond - with the working tree. Additionally, the cache must be cleared. + Otherwise it would add the local file to the index to make it correspond with the + working tree. Additionally, the cache must be cleared. Please note that no mutating method will work in bare mode. """ @@ -86,8 +87,8 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: # { Interface def set_submodule(self, submodule: "Submodule") -> None: - """Set this instance's submodule. It must be called before - the first write operation begins.""" + """Set this instance's submodule. It must be called before the first write + operation begins.""" self._smref = weakref.ref(submodule) def flush_to_index(self) -> None: From 115451d4605b880e26c59bb415b539cdce984b64 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 26 Feb 2024 20:13:16 -0500 Subject: [PATCH 0679/1392] Change _write to write in SubmoduleConfigParser docstring Since it appears to refer to the write method, which it overrides, rather that the nonpublic _write method (which is not overridden, and whose direct calls are not affected). --- git/objects/submodule/util.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index 91e18a26b..b02da501e 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -70,8 +70,9 @@ def find_first_remote_branch(remotes: Sequence["Remote"], branch_name: str) -> " class SubmoduleConfigParser(GitConfigParser): - """Catches calls to _write, and updates the ``.gitmodules`` blob in the index - with the new data, if we have written into a stream. + """Catches calls to :meth:`~git.config.GitConfigParser.write`, and updates the + ``.gitmodules`` blob in the index with the new data, if we have written into a + stream. Otherwise it would add the local file to the index to make it correspond with the working tree. Additionally, the cache must be cleared. From 521948989134c8e0ab7d4c1063780a0799f4dbc8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 26 Feb 2024 20:58:21 -0500 Subject: [PATCH 0680/1392] Fix IndexObject.abspath docstring formatting The original problem where the backslash wasn't included in the docstring at all was fixed in 7dd2095 (#1725), but the backslash still did not appear in rendered Sphinx documentation, because it was also treated as a reStructuredText metacharacter. Although that can be addressed by adding a further backslash to escape it, the effect is ambiguous when the docstring is read in the code. So this just encloses it in a double-backticked code span instead, which is a somewhat clearer way to show it anyway. --- git/objects/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/git/objects/base.py b/git/objects/base.py index 934fb40bc..938dd826c 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -226,7 +226,8 @@ def abspath(self) -> PathLike: Absolute path to this index object in the file system (as opposed to the :attr:`path` field which is a path relative to the git repository). - The returned path will be native to the system and contains '\' on Windows. + The returned path will be native to the system and contains ``\`` on + Windows. """ if self.repo.working_tree_dir is not None: return join_path_native(self.repo.working_tree_dir, self.path) From c06dfd9bdc076e44df436836914d07a1f9f695c1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 26 Feb 2024 21:49:29 -0500 Subject: [PATCH 0681/1392] Fix parameter names in TagObject.__init__ --- git/objects/tag.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/objects/tag.py b/git/objects/tag.py index f7aaaf477..f874f5bdf 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -59,8 +59,8 @@ def __init__( :param tagged_date: int_seconds_since_epoch The :class:`DateTime` of the tag creation. Use :func:`time.gmtime` to convert it into a different format. - :param tagged_tz_offset: int_seconds_west_of_utc - The timezone that the authored_date is in, in a format similar + :param tagger_tz_offset: int_seconds_west_of_utc + The timezone that the tagged_date is in, in a format similar to :attr:`time.altzone`. """ super().__init__(repo, binsha) From ae37a4a699608c6bc55e023547c8daa473477920 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 13:21:46 -0500 Subject: [PATCH 0682/1392] Revise docstrings within git.objects But not within git.objects.submodule, which was done in 63c62ed. --- git/objects/__init__.py | 4 +- git/objects/base.py | 71 +++++++---- git/objects/blob.py | 3 +- git/objects/commit.py | 252 ++++++++++++++++++++++++---------------- git/objects/fun.py | 69 ++++++----- git/objects/tag.py | 28 +++-- git/objects/tree.py | 88 ++++++++------ git/objects/util.py | 167 +++++++++++++++----------- 8 files changed, 415 insertions(+), 267 deletions(-) diff --git a/git/objects/__init__.py b/git/objects/__init__.py index fc8d8fbb2..1061ec874 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -14,8 +14,8 @@ from .tag import * # noqa: F403 from .tree import * # noqa: F403 -# Fix import dependency - add IndexObject to the util module, so that it can be -# imported by the submodule.base. +# Fix import dependency - add IndexObject to the util module, so that it can be imported +# by the submodule.base. smutil.IndexObject = IndexObject # type: ignore[attr-defined] # noqa: F405 smutil.Object = Object # type: ignore[attr-defined] # noqa: F405 del smutil diff --git a/git/objects/base.py b/git/objects/base.py index 938dd826c..df56a4bac 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -37,7 +37,9 @@ class Object(LazyMixin): - """An Object which may be Blobs, Trees, Commits and Tags.""" + """An Object which may be :class:`~git.objects.blob.Blob`, + :class:`~git.objects.tree.Tree`, :class:`~git.objects.commit.Commit` or + `~git.objects.tag.TagObject`.""" NULL_HEX_SHA = "0" * 40 NULL_BIN_SHA = b"\0" * 20 @@ -55,11 +57,14 @@ class Object(LazyMixin): def __init__(self, repo: "Repo", binsha: bytes): """Initialize an object by identifying it by its binary sha. + All keyword arguments will be set on demand if None. - :param repo: repository this object is located in + :param repo: + Repository this object is located in. - :param binsha: 20 byte SHA1 + :param binsha: + 20 byte SHA1 """ super().__init__() self.repo = repo @@ -72,24 +77,28 @@ def __init__(self, repo: "Repo", binsha: bytes): @classmethod def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> Commit_ish: """ - :return: New :class:`Object`` instance of a type appropriate to the object type - behind `id`. The id of the newly created object will be a binsha even though - the input id may have been a Reference or Rev-Spec. + :return: + New :class:`Object` instance of a type appropriate to the object type behind + `id`. The id of the newly created object will be a binsha even though the + input id may have been a `~git.refs.reference.Reference` or rev-spec. - :param id: reference, rev-spec, or hexsha + :param id: + :class:`~git.refs.reference.Reference`, rev-spec, or hexsha - :note: This cannot be a ``__new__`` method as it would always call - :meth:`__init__` with the input id which is not necessarily a binsha. + :note: + This cannot be a ``__new__`` method as it would always call :meth:`__init__` + with the input id which is not necessarily a binsha. """ return repo.rev_parse(str(id)) @classmethod def new_from_sha(cls, repo: "Repo", sha1: bytes) -> Commit_ish: """ - :return: new object instance of a type appropriate to represent the given - binary sha1 + :return: + New object instance of a type appropriate to represent the given binary sha1 - :param sha1: 20 byte binary sha1 + :param sha1: + 20 byte binary sha1 """ if sha1 == cls.NULL_BIN_SHA: # The NULL binsha is always the root commit. @@ -126,11 +135,11 @@ def __hash__(self) -> int: return hash(self.binsha) def __str__(self) -> str: - """:return: string of our SHA1 as understood by all git commands""" + """:return: String of our SHA1 as understood by all git commands""" return self.hexsha def __repr__(self) -> str: - """:return: string with pythonic representation of our object""" + """:return: String with pythonic representation of our object""" return '' % (self.__class__.__name__, self.hexsha) @property @@ -142,17 +151,22 @@ def hexsha(self) -> str: @property def data_stream(self) -> "OStream": """ - :return: File Object compatible stream to the uncompressed raw data of the object + :return: + File-object compatible stream to the uncompressed raw data of the object - :note: Returned streams must be read in order. + :note: + Returned streams must be read in order. """ return self.repo.odb.stream(self.binsha) def stream_data(self, ostream: "OStream") -> "Object": """Write our data directly to the given output stream. - :param ostream: File object compatible stream object. - :return: self + :param ostream: + File-object compatible stream object. + + :return: + self """ istream = self.repo.odb.stream(self.binsha) stream_copy(istream, ostream) @@ -160,8 +174,9 @@ def stream_data(self, ostream: "OStream") -> "Object": class IndexObject(Object): - """Base for all objects that can be part of the index file, namely Tree, Blob and - SubModule objects.""" + """Base for all objects that can be part of the index file, namely + :class:`~git.objects.tree.Tree`, :class:`~git.objects.blob.Blob` and + :class:`~git.objects.submodule.base.Submodule` objects.""" __slots__ = ("path", "mode") @@ -175,16 +190,22 @@ def __init__( mode: Union[None, int] = None, path: Union[None, PathLike] = None, ) -> None: - """Initialize a newly instanced IndexObject. + """Initialize a newly instanced :class:`IndexObject`. + + :param repo: + The :class:`~git.repo.base.Repo` we are located in. + + :param binsha: + 20 byte sha1. - :param repo: The :class:`~git.repo.base.Repo` we are located in. - :param binsha: 20 byte sha1. :param mode: The stat compatible file mode as int, use the :mod:`stat` module to evaluate the information. + :param path: The path to the file in the file system, relative to the git repository root, like ``file.ext`` or ``folder/other.ext``. + :note: Path may not be set if the index object has been created directly, as it cannot be retrieved without knowing the parent tree. @@ -198,8 +219,8 @@ def __init__( def __hash__(self) -> int: """ :return: - Hash of our path as index items are uniquely identifiable by path, not - by their data! + Hash of our path as index items are uniquely identifiable by path, not by + their data! """ return hash(self.path) diff --git a/git/objects/blob.py b/git/objects/blob.py index 6d7e859af..50c12f0d7 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -29,7 +29,8 @@ def mime_type(self) -> str: """ :return: String describing the mime type of this file (based on the filename) - :note: Defaults to 'text/plain' in case the actual file type is unknown. + :note: + Defaults to ``text/plain`` in case the actual file type is unknown. """ guesses = None if self.path: diff --git a/git/objects/commit.py b/git/objects/commit.py index caec1b6c4..e6aa7ac39 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -58,11 +58,11 @@ class Commit(base.Object, TraversableIterableObj, Diffable, Serializable): + """Wraps a git commit object. - """Wraps a git Commit object. - - This class will act lazily on some of its attributes and will query the - value on demand only if it involves calling the git binary.""" + This class will act lazily on some of its attributes and will query the value on + demand only if it involves calling the git binary. + """ # ENVIRONMENT VARIABLES # Read when creating new commits. @@ -108,41 +108,55 @@ def __init__( encoding: Union[str, None] = None, gpgsig: Union[str, None] = None, ) -> None: - """Instantiate a new Commit. All keyword arguments taking None as default will - be implicitly set on first query. + R"""Instantiate a new :class:`Commit`. All keyword arguments taking None as + default will be implicitly set on first query. + + :param binsha: + 20 byte sha1 - :param binsha: 20 byte sha1 :param parents: tuple(Commit, ...) - A tuple of commit ids or actual Commits - :param tree: Tree object - :param author: Actor + A tuple of commit ids or actual :class:`Commit`\s. + + :param tree: + A :class:`~git.objects.tree.Tree` object. + + :param author: :class:`~git.util.Actor` The author Actor object + :param authored_date: int_seconds_since_epoch - The authored DateTime - use time.gmtime() to convert it into a - different format + The authored DateTime - use :func:`time.gmtime` to convert it into a + different format. + :param author_tz_offset: int_seconds_west_of_utc - The timezone that the authored_date is in - :param committer: Actor - The committer string + The timezone that the `authored_date` is in. + + :param committer: :class:`~git.util.Actor` + The committer string. + :param committed_date: int_seconds_since_epoch - The committed DateTime - use time.gmtime() to convert it into a - different format + The committed DateTime - use :func:`time.gmtime` to convert it into a + different format. + :param committer_tz_offset: int_seconds_west_of_utc - The timezone that the committed_date is in + The timezone that the `committed_date` is in. + :param message: string - The commit message + The commit message. + :param encoding: string - Encoding of the message, defaults to UTF-8 + Encoding of the message, defaults to UTF-8. + :param parents: - List or tuple of Commit objects which are our parent(s) in the commit - dependency graph + List or tuple of :class:`Commit` objects which are our parent(s) in the + commit dependency graph. - :return: git.Commit + :return: + :class:`Commit` :note: - Timezone information is in the same format and in the same sign - as what time.altzone returns. The sign is inverted compared to git's - UTC timezone. + Timezone information is in the same format and in the same sign as what + :func:`time.altzone` returns. The sign is inverted compared to git's UTC + timezone. """ super().__init__(repo, binsha) self.binsha = binsha @@ -179,8 +193,11 @@ def _get_intermediate_items(cls, commit: "Commit") -> Tuple["Commit", ...]: def _calculate_sha_(cls, repo: "Repo", commit: "Commit") -> bytes: """Calculate the sha of a commit. - :param repo: Repo object the commit should be part of - :param commit: Commit object for which to generate the sha + :param repo: + :class:`~git.repo.base.Repo` object the commit should be part of. + + :param commit: + :class:`Commit` object for which to generate the sha. """ stream = BytesIO() @@ -194,8 +211,8 @@ def _calculate_sha_(cls, repo: "Repo", commit: "Commit") -> bytes: def replace(self, **kwargs: Any) -> "Commit": """Create new commit object from existing commit object. - Any values provided as keyword arguments will replace the - corresponding attribute in the new object. + Any values provided as keyword arguments will replace the corresponding + attribute in the new object. """ attrs = {k: getattr(self, k) for k in self.__slots__} @@ -239,17 +256,18 @@ def count(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) """Count the number of commits reachable from this commit. :param paths: - An optional path or a list of paths restricting the return value - to commits actually containing the paths. + An optional path or a list of paths restricting the return value to commits + actually containing the paths. :param kwargs: - Additional options to be passed to git-rev-list. They must not alter - the output style of the command, or parsing will yield incorrect results. + Additional options to be passed to ``git rev-list``. They must not alter the + output style of the command, or parsing will yield incorrect results. - :return: An int defining the number of reachable commits + :return: + An int defining the number of reachable commits """ - # Yes, it makes a difference whether empty paths are given or not in our case - # as the empty paths version will ignore merge commits for some reason. + # Yes, it makes a difference whether empty paths are given or not in our case as + # the empty paths version will ignore merge commits for some reason. if paths: return len(self.repo.git.rev_list(self.hexsha, "--", paths, **kwargs).splitlines()) return len(self.repo.git.rev_list(self.hexsha, **kwargs).splitlines()) @@ -258,8 +276,11 @@ def count(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) def name_rev(self) -> str: """ :return: - String describing the commits hex sha based on the closest Reference. - Mostly useful for UI purposes + String describing the commits hex sha based on the closest + `~git.refs.reference.Reference`. + + :note: + Mostly useful for UI purposes. """ return self.repo.git.name_rev(self) @@ -271,24 +292,27 @@ def iter_items( paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any, ) -> Iterator["Commit"]: - """Find all commits matching the given criteria. + R"""Find all commits matching the given criteria. - :param repo: The Repo + :param repo: + The Repo - :param rev: Revision specifier, see git-rev-parse for viable options. + :param rev: + Revision specifier, see git-rev-parse for viable options. :param paths: - An optional path or list of paths, if set only Commits that include the path - or paths will be considered. + An optional path or list of paths. If set only :class:`Commit`\s that + include the path or paths will be considered. :param kwargs: Optional keyword arguments to ``git rev-list`` where: - * ``max_count`` is the maximum number of commits to fetch - * ``skip`` is the number of commits to skip - * ``since`` all commits since e.g. '1970-01-01' + * ``max_count`` is the maximum number of commits to fetch. + * ``skip`` is the number of commits to skip. + * ``since`` all commits since e.g. '1970-01-01'. - :return: Iterator yielding :class:`Commit` items. + :return: + Iterator yielding :class:`Commit` items. """ if "pretty" in kwargs: raise ValueError("--pretty cannot be used as parsing expects single sha's only") @@ -313,13 +337,17 @@ def iter_items( return cls._iter_from_process_or_stream(repo, proc) def iter_parents(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) -> Iterator["Commit"]: - """Iterate _all_ parents of this commit. + R"""Iterate _all_ parents of this commit. :param paths: - Optional path or list of paths limiting the Commits to those that - contain at least one of the paths - :param kwargs: All arguments allowed by git-rev-list - :return: Iterator yielding Commit objects which are parents of self + Optional path or list of paths limiting the :class:`Commit`\s to those that + contain at least one of the paths. + + :param kwargs: + All arguments allowed by ``git rev-list``. + + :return: + Iterator yielding :class:`Commit` objects which are parents of ``self`` """ # skip ourselves skip = kwargs.get("skip", 1) @@ -334,7 +362,8 @@ def stats(self) -> Stats: """Create a git stat from changes between this commit and its first parent or from all changes done if this is the very first commit. - :return: git.Stats + :return: + :class:`Stats` """ if not self.parents: text = self.repo.git.diff_tree(self.hexsha, "--", numstat=True, no_renames=True, root=True) @@ -364,11 +393,11 @@ def trailers(self) -> Dict[str, str]: def trailers_list(self) -> List[Tuple[str, str]]: """Get the trailers of the message as a list. - Git messages can contain trailer information that are similar to RFC 822 - e-mail headers (see: https://git-scm.com/docs/git-interpret-trailers). + Git messages can contain trailer information that are similar to RFC 822 e-mail + headers (see: https://git-scm.com/docs/git-interpret-trailers). - This functions calls ``git interpret-trailers --parse`` onto the message - to extract the trailer information, returns the raw trailer data as a list. + This functions calls ``git interpret-trailers --parse`` onto the message to + extract the trailer information, returns the raw trailer data as a list. Valid message with trailer:: @@ -382,7 +411,6 @@ def trailers_list(self) -> List[Tuple[str, str]]: key1: value1.2 key2 : value 2 with inner spaces - Returned list will look like this:: [ @@ -391,7 +419,6 @@ def trailers_list(self) -> List[Tuple[str, str]]: ("key2", "value 2 with inner spaces"), ] - :return: List containing key-value tuples of whitespace stripped trailer information. """ @@ -414,12 +441,12 @@ def trailers_list(self) -> List[Tuple[str, str]]: def trailers_dict(self) -> Dict[str, List[str]]: """Get the trailers of the message as a dictionary. - Git messages can contain trailer information that are similar to RFC 822 - e-mail headers (see: https://git-scm.com/docs/git-interpret-trailers). + Git messages can contain trailer information that are similar to RFC 822 e-mail + headers (see: https://git-scm.com/docs/git-interpret-trailers). - This functions calls ``git interpret-trailers --parse`` onto the message - to extract the trailer information. The key value pairs are stripped of - leading and trailing whitespaces before they get saved into a dictionary. + This functions calls ``git interpret-trailers --parse`` onto the message to + extract the trailer information. The key value pairs are stripped of leading and + trailing whitespaces before they get saved into a dictionary. Valid message with trailer:: @@ -433,7 +460,6 @@ def trailers_dict(self) -> Dict[str, List[str]]: key1: value1.2 key2 : value 2 with inner spaces - Returned dictionary will look like this:: { @@ -443,8 +469,8 @@ def trailers_dict(self) -> Dict[str, List[str]]: :return: - Dictionary containing whitespace stripped trailer information. - Mapping trailer keys to a list of their corresponding values. + Dictionary containing whitespace stripped trailer information, mapping + trailer keys to a list of their corresponding values. """ d = defaultdict(list) for key, val in self.trailers_list: @@ -453,13 +479,16 @@ def trailers_dict(self) -> Dict[str, List[str]]: @classmethod def _iter_from_process_or_stream(cls, repo: "Repo", proc_or_stream: Union[Popen, IO]) -> Iterator["Commit"]: - """Parse out commit information into a list of Commit objects. + """Parse out commit information into a list of :class:`Commit` objects. We expect one-line per commit, and parse the actual commit information directly from our lighting fast object database. - :param proc: git-rev-list process instance - one sha per line - :return: iterator supplying :class:`Commit` objects + :param proc: + ``git rev-list`` process instance - one sha per line. + + :return: + Iterator supplying :class:`Commit` objects """ # def is_proc(inp) -> TypeGuard[Popen]: @@ -491,8 +520,8 @@ def _iter_from_process_or_stream(cls, repo: "Repo", proc_or_stream: Union[Popen, yield cls(repo, hex_to_bin(hexsha)) # END for each line in stream - # TODO: Review this - it seems process handling got a bit out of control - # due to many developers trying to fix the open file handles issue. + # TODO: Review this - it seems process handling got a bit out of control due to + # many developers trying to fix the open file handles issue. if hasattr(proc_or_stream, "wait"): proc_or_stream = cast(Popen, proc_or_stream) finalize_process(proc_or_stream) @@ -512,33 +541,49 @@ def create_from_tree( ) -> "Commit": """Commit the given tree, creating a commit object. - :param repo: Repo object the commit should be part of - :param tree: Tree object or hex or bin sha. The tree of the new commit. - :param message: Commit message. It may be an empty string if no message is - provided. It will be converted to a string, in any case. + :param repo: + :class:`~git.repo.base.Repo` object the commit should be part of. + + :param tree: + :class:`~git.objects.tree.Tree` object or hex or bin sha. + The tree of the new commit. + + :param message: + Commit message. It may be an empty string if no message is provided. It will + be converted to a string, in any case. + :param parent_commits: - Optional :class:`Commit` objects to use as parents for the new commit. - If empty list, the commit will have no parents at all and become - a root commit. - If None, the current head commit will be the parent of the - new commit object. + Optional :class:`Commit` objects to use as parents for the new commit. If + empty list, the commit will have no parents at all and become a root commit. + If None, the current head commit will be the parent of the new commit + object. + :param head: If True, the HEAD will be advanced to the new commit automatically. Otherwise the HEAD will remain pointing on the previous commit. This could lead to undesired results when diffing files. - :param author: The name of the author, optional. If unset, the repository - configuration is used to obtain this value. - :param committer: The name of the committer, optional. If unset, the - repository configuration is used to obtain this value. - :param author_date: The timestamp for the author field. - :param commit_date: The timestamp for the committer field. - :return: Commit object representing the new commit. + :param author: + The name of the author, optional. + If unset, the repository configuration is used to obtain this value. + + :param committer: + The name of the committer, optional. + If unset, the repository configuration is used to obtain this value. + + :param author_date: + The timestamp for the author field. + + :param commit_date: + The timestamp for the committer field. + + :return: + :class:`Commit` object representing the new commit. :note: - Additional information about the committer and Author are taken from the - environment or from the git configuration, see git-commit-tree for - more information. + Additional information about the committer and author are taken from the + environment or from the git configuration. See git-commit-tree for more + information. """ if parent_commits is None: try: @@ -595,8 +640,8 @@ def create_from_tree( if not isinstance(conf_encoding, str): raise TypeError("conf_encoding could not be coerced to str") - # If the tree is no object, make sure we create one - otherwise - # the created commit object is invalid. + # If the tree is no object, make sure we create one - otherwise the created + # commit object is invalid. if isinstance(tree, str): tree = repo.tree(tree) # END tree conversion @@ -620,8 +665,8 @@ def create_from_tree( new_commit.binsha = cls._calculate_sha_(repo, new_commit) if head: - # Need late import here, importing git at the very beginning throws - # as well... + # Need late import here, importing git at the very beginning throws as + # well... import git.refs try: @@ -718,7 +763,8 @@ def _deserialize(self, stream: BytesIO) -> "Commit": # END for each parent line self.parents = tuple(self.parents) - # We don't know actual author encoding before we have parsed it, so keep the lines around. + # We don't know actual author encoding before we have parsed it, so keep the + # lines around. author_line = next_line committer_line = readline() @@ -730,7 +776,8 @@ def _deserialize(self, stream: BytesIO) -> "Commit": next_line = readline() # END skip mergetags - # Now we can have the encoding line, or an empty line followed by the optional message. + # Now we can have the encoding line, or an empty line followed by the optional + # message. self.encoding = self.default_encoding self.gpgsig = "" @@ -808,12 +855,13 @@ def _deserialize(self, stream: BytesIO) -> "Commit": @property def co_authors(self) -> List[Actor]: - """ - Search the commit message for any co-authors of this commit. + """Search the commit message for any co-authors of this commit. - Details on co-authors: https://github.blog/2018-01-29-commit-together-with-co-authors/ + Details on co-authors: + https://github.blog/2018-01-29-commit-together-with-co-authors/ - :return: List of co-authors for this commit (as Actor objects). + :return: + List of co-authors for this commit (as :class:`~git.util.Actor` objects). """ co_authors = [] diff --git a/git/objects/fun.py b/git/objects/fun.py index bf6bc21d6..d349737de 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -40,10 +40,13 @@ def tree_to_stream(entries: Sequence[EntryTup], write: Callable[["ReadableBuffer"], Union[int, None]]) -> None: - """Write the given list of entries into a stream using its write method. + """Write the given list of entries into a stream using its ``write`` method. - :param entries: **sorted** list of tuples with (binsha, mode, name) - :param write: write method which takes a data string + :param entries: + **Sorted** list of tuples with (binsha, mode, name). + + :param write: + A ``write`` method which takes a data string. """ ord_zero = ord("0") bit_mask = 7 # 3 bits set. @@ -59,11 +62,11 @@ def tree_to_stream(entries: Sequence[EntryTup], write: Callable[["ReadableBuffer mode_str = mode_str[1:] # END save a byte - # Here it comes: If the name is actually unicode, the replacement below - # will not work as the binsha is not part of the ascii unicode encoding - - # hence we must convert to an UTF-8 string for it to work properly. - # According to my tests, this is exactly what git does, that is it just - # takes the input literally, which appears to be UTF-8 on linux. + # Here it comes: If the name is actually unicode, the replacement below will not + # work as the binsha is not part of the ascii unicode encoding - hence we must + # convert to an UTF-8 string for it to work properly. According to my tests, + # this is exactly what git does, that is it just takes the input literally, + # which appears to be UTF-8 on linux. if isinstance(name, str): name_bytes = name.encode(defenc) else: @@ -73,10 +76,15 @@ def tree_to_stream(entries: Sequence[EntryTup], write: Callable[["ReadableBuffer def tree_entries_from_data(data: bytes) -> List[EntryTup]: - """Reads the binary representation of a tree and returns tuples of Tree items + """Read the binary representation of a tree and returns tuples of + :class:`~git.objects.tree.Tree` items. + + :param data: + Data block with tree data (as bytes). - :param data: data block with tree data (as bytes) - :return: list(tuple(binsha, mode, tree_relative_path), ...)""" + :return: + list(tuple(binsha, mode, tree_relative_path), ...) + """ ord_zero = ord("0") space_ord = ord(" ") len_data = len(data) @@ -89,8 +97,8 @@ def tree_entries_from_data(data: bytes) -> List[EntryTup]: # Some git versions truncate the leading 0, some don't. # The type will be extracted from the mode later. while data[i] != space_ord: - # Move existing mode integer up one level being 3 bits - # and add the actual ordinal value of the character. + # Move existing mode integer up one level being 3 bits and add the actual + # ordinal value of the character. mode = (mode << 3) + (data[i] - ord_zero) i += 1 # END while reading mode @@ -122,8 +130,8 @@ def tree_entries_from_data(data: bytes) -> List[EntryTup]: def _find_by_name(tree_data: MutableSequence[EntryTupOrNone], name: str, is_dir: bool, start_at: int) -> EntryTupOrNone: """Return data entry matching the given name and tree mode or None. - Before the item is returned, the respective data item is set None in the - tree_data list to mark it done. + Before the item is returned, the respective data item is set None in the `tree_data` + list to mark it done. """ try: @@ -165,6 +173,7 @@ def traverse_trees_recursive( ) -> List[Tuple[EntryTupOrNone, ...]]: """ :return: list of list with entries according to the given binary tree-shas. + The result is encoded in a list of n tuple|None per blob/commit, (n == len(tree_shas)), where: @@ -172,16 +181,19 @@ def traverse_trees_recursive( * [1] == mode as int * [2] == path relative to working tree root - The entry tuple is None if the respective blob/commit did not - exist in the given tree. + The entry tuple is None if the respective blob/commit did not exist in the given + tree. - :param tree_shas: iterable of shas pointing to trees. All trees must - be on the same level. A tree-sha may be None in which case None. + :param tree_shas: + Iterable of shas pointing to trees. All trees must be on the same level. A + tree-sha may be None in which case None. - :param path_prefix: a prefix to be added to the returned paths on this level, - set it '' for the first iteration. + :param path_prefix: + A prefix to be added to the returned paths on this level. + Set it ``""`` for the first iteration. - :note: The ordering of the returned items will be partially lost. + :note: + The ordering of the returned items will be partially lost. """ trees_data: List[List[EntryTupOrNone]] = [] @@ -198,8 +210,8 @@ def traverse_trees_recursive( out: List[Tuple[EntryTupOrNone, ...]] = [] - # Find all matching entries and recursively process them together if the match - # is a tree. If the match is a non-tree item, put it into the result. + # Find all matching entries and recursively process them together if the match is a + # tree. If the match is a non-tree item, put it into the result. # Processed items will be set None. for ti, tree_data in enumerate(trees_data): for ii, item in enumerate(tree_data): @@ -213,8 +225,8 @@ def traverse_trees_recursive( is_dir = S_ISDIR(mode) # Type mode bits # Find this item in all other tree data items. - # Wrap around, but stop one before our current index, hence - # ti+nt, not ti+1+nt. + # Wrap around, but stop one before our current index, hence ti+nt, not + # ti+1+nt. for tio in range(ti + 1, ti + nt): tio = tio % nt entries[tio] = _find_by_name(trees_data[tio], name, is_dir, ii) @@ -245,7 +257,7 @@ def traverse_trees_recursive( def traverse_tree_recursive(odb: "GitCmdObjectDB", tree_sha: bytes, path_prefix: str) -> List[EntryTup]: """ - :return: list of entries of the tree pointed to by the binary tree_sha. + :return: list of entries of the tree pointed to by the binary `tree_sha`. An entry has the following format: @@ -253,7 +265,8 @@ def traverse_tree_recursive(odb: "GitCmdObjectDB", tree_sha: bytes, path_prefix: * [1] mode as int * [2] path relative to the repository - :param path_prefix: Prefix to prepend to the front of all returned paths. + :param path_prefix: + Prefix to prepend to the front of all returned paths. """ entries = [] data = tree_entries_from_data(odb.stream(tree_sha).read()) diff --git a/git/objects/tag.py b/git/objects/tag.py index f874f5bdf..211c84ac7 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -25,7 +25,8 @@ class TagObject(base.Object): - """Non-lightweight tag carrying additional information about an object we are pointing to.""" + """Non-lightweight tag carrying additional information about an object we are + pointing to.""" type: Literal["tag"] = "tag" @@ -51,17 +52,28 @@ def __init__( ) -> None: # @ReservedAssignment """Initialize a tag object with additional data. - :param repo: Repository this object is located in - :param binsha: 20 byte SHA1 - :param object: Object instance of object we are pointing to - :param tag: Name of this tag - :param tagger: Actor identifying the tagger + :param repo: + Repository this object is located in. + + :param binsha: + 20 byte SHA1. + + :param object: + :class:`~git.objects.base.Object` instance of object we are pointing to. + + :param tag: + Name of this tag. + + :param tagger: + :class:`~git.util.Actor` identifying the tagger. + :param tagged_date: int_seconds_since_epoch The :class:`DateTime` of the tag creation. Use :func:`time.gmtime` to convert it into a different format. + :param tagger_tz_offset: int_seconds_west_of_utc - The timezone that the tagged_date is in, in a format similar - to :attr:`time.altzone`. + The timezone that the `tagged_date` is in, in a format similar to + :attr:`time.altzone`. """ super().__init__(repo, binsha) if object is not None: diff --git a/git/objects/tree.py b/git/objects/tree.py index 450973280..da682b8cd 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -54,10 +54,12 @@ class TreeModifier: - """A utility class providing methods to alter the underlying cache in a list-like fashion. + """A utility class providing methods to alter the underlying cache in a list-like + fashion. - Once all adjustments are complete, the _cache, which really is a reference to - the cache of a tree, will be sorted. This ensures it will be in a serializable state. + Once all adjustments are complete, the :attr:`_cache`, which really is a reference + to the cache of a tree, will be sorted. This ensures it will be in a serializable + state. """ __slots__ = ("_cache",) @@ -78,10 +80,11 @@ def _index_by_name(self, name: str) -> int: def set_done(self) -> "TreeModifier": """Call this method once you are done modifying the tree information. - This may be called several times, but be aware that each call will cause - a sort operation. + This may be called several times, but be aware that each call will cause a sort + operation. - :return self: + :return: + self """ self._cache.sort(key=lambda x: (x[2] + "/") if x[1] == Tree.tree_id << 12 else x[2]) return self @@ -93,17 +96,21 @@ def add(self, sha: bytes, mode: int, name: str, force: bool = False) -> "TreeMod """Add the given item to the tree. If an item with the given name already exists, nothing will be done, but a - ValueError will be raised if the sha and mode of the existing item do not match - the one you add, unless force is True + :class:`ValueError` will be raised if the sha and mode of the existing item do + not match the one you add, unless `force` is True. - :param sha: The 20 or 40 byte sha of the item to add + :param sha: + The 20 or 40 byte sha of the item to add. - :param mode: int representing the stat compatible mode of the item + :param mode: + int representing the stat compatible mode of the item. - :param force: If True, an item with your name and information will overwrite any - existing item with the same name, no matter which information it has + :param force: + If True, an item with your name and information will overwrite any existing + item with the same name, no matter which information it has. - :return: self + :return: + self """ if "/" in name: raise ValueError("Name must not contain '/' characters") @@ -136,7 +143,8 @@ def add_unchecked(self, binsha: bytes, mode: int, name: str) -> None: For more information on the parameters, see :meth:`add`. - :param binsha: 20 byte binary sha + :param binsha: + 20 byte binary sha. """ assert isinstance(binsha, bytes) and isinstance(mode, int) and isinstance(name, str) tree_cache = (binsha, mode, name) @@ -153,15 +161,14 @@ def __delitem__(self, name: str) -> None: class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): - """Tree objects represent an ordered list of Blobs and other Trees. + R"""Tree objects represent an ordered list of :class:`~git.objects.blob.Blob`\s and + other :class:`~git.objects.tree.Tree`\s. - ``Tree as a list``:: + Tree as a list:: - Access a specific blob using the - tree['filename'] notation. + Access a specific blob using the ``tree['filename']`` notation. - You may as well access by index - blob = tree[0] + You may likewise access by index, like ``blob = tree[0]``. """ type: Literal["tree"] = "tree" @@ -223,8 +230,12 @@ def _iter_convert_to_object(self, iterable: Iterable[TreeCacheTup]) -> Iterator[ def join(self, file: str) -> IndexObjUnion: """Find the named object in this tree's contents. - :return: ``git.Blob`` or ``git.Tree`` or ``git.Submodule`` - :raise KeyError: if given file or tree does not exist in tree + :return: + :class:`~git.objects.blob.Blob`, :class:`~git.objects.tree.Tree`, + or :class:`~git.objects.submodule.base.Submodule` + + :raise KeyError: + If the given file or tree does not exist in tree. """ msg = "Blob or Tree named %r not found" if "/" in file: @@ -275,9 +286,13 @@ def blobs(self) -> List[Blob]: @property def cache(self) -> TreeModifier: """ - :return: An object allowing to modify the internal cache. This can be used - to change the tree's contents. When done, make sure you call ``set_done`` - on the tree modifier, or serialization behaviour will be incorrect. + :return: + An object allowing modification of the internal cache. This can be used to + change the tree's contents. When done, make sure you call + :meth:`~TreeModifier.set_done` on the tree modifier, or serialization + behaviour will be incorrect. + + :note: See :class:`TreeModifier` for more information on how to alter the cache. """ return TreeModifier(self._cache) @@ -292,12 +307,13 @@ def traverse( ignore_self: int = 1, as_edge: bool = False, ) -> Union[Iterator[IndexObjUnion], Iterator[TraversedTreeTup]]: - """For documentation, see util.Traversable._traverse(). + """For documentation, see + `Traversable._traverse() `. - Trees are set to ``visit_once = False`` to gain more performance in the traversal. + Trees are set to ``visit_once = False`` to gain more performance in the + traversal. """ - # """ # # To typecheck instead of using cast. # import itertools # def is_tree_traversed(inp: Tuple) -> TypeGuard[Tuple[Iterator[Union['Tree', 'Blob', 'Submodule']]]]: @@ -306,7 +322,8 @@ def traverse( # ret = super().traverse(predicate, prune, depth, branch_first, visit_once, ignore_self) # ret_tup = itertools.tee(ret, 2) # assert is_tree_traversed(ret_tup), f"Type is {[type(x) for x in list(ret_tup[0])]}" - # return ret_tup[0]""" + # return ret_tup[0] + return cast( Union[Iterator[IndexObjUnion], Iterator[TraversedTreeTup]], super()._traverse( @@ -321,8 +338,10 @@ def traverse( def list_traverse(self, *args: Any, **kwargs: Any) -> IterableList[IndexObjUnion]: """ - :return: IterableList with the results of the traversal as produced by - traverse() + :return: + :class:`~git.util.IterableList`IterableList with the results of the + traversal as produced by :meth:`traverse` + Tree -> IterableList[Union['Submodule', 'Tree', 'Blob']] """ return super()._list_traverse(*args, **kwargs) @@ -375,9 +394,10 @@ def __reversed__(self) -> Iterator[IndexObjUnion]: def _serialize(self, stream: "BytesIO") -> "Tree": """Serialize this tree into the stream. Assumes sorted tree data. - .. note:: We will assume our tree data to be in a sorted state. If this is not - the case, serialization will not generate a correct tree representation as - these are assumed to be sorted by algorithms. + :note: + We will assume our tree data to be in a sorted state. If this is not the + case, serialization will not generate a correct tree representation as these + are assumed to be sorted by algorithms. """ tree_to_stream(self._cache, stream.write) return self diff --git a/git/objects/util.py b/git/objects/util.py index b25a3f5ff..7ad20d66e 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -88,14 +88,16 @@ class TraverseNT(NamedTuple): def mode_str_to_int(modestr: Union[bytes, str]) -> int: - """ + """Convert mode bits from an octal mode string to an integer mode for git. + :param modestr: - String like 755 or 644 or 100644 - only the last 6 chars will be used. + String like ``755`` or ``644`` or ``100644`` - only the last 6 chars will be + used. :return: - String identifying a mode compatible to the mode methods ids of the - stat module regarding the rwx permissions for user, group and other, - special flags and file system flags, such as whether it is a symlink. + String identifying a mode compatible to the mode methods ids of the stat module + regarding the rwx permissions for user, group and other, special flags and file + system flags, such as whether it is a symlink. """ mode = 0 for iteration, char in enumerate(reversed(modestr[-6:])): @@ -108,13 +110,17 @@ def mode_str_to_int(modestr: Union[bytes, str]) -> int: def get_object_type_by_name( object_type_name: bytes, ) -> Union[Type["Commit"], Type["TagObject"], Type["Tree"], Type["Blob"]]: - """ - :return: A type suitable to handle the given object type name. - Use the type to create new instances. + """Retrieve the Python class GitPython uses to represent a kind of Git object. + + :return: + A type suitable to handle the given as `object_type_name`. + This type can be called create new instances. - :param object_type_name: Member of TYPES + :param object_type_name: + Member of :attr:`Object.TYPES `. - :raise ValueError: If object_type_name is unknown + :raise ValueError: + If `object_type_name` is unknown. """ if object_type_name == b"commit": from . import commit @@ -137,10 +143,11 @@ def get_object_type_by_name( def utctz_to_altz(utctz: str) -> int: - """Convert a git timezone offset into a timezone offset west of - UTC in seconds (compatible with :attr:`time.altzone`). + """Convert a git timezone offset into a timezone offset west of UTC in seconds + (compatible with :attr:`time.altzone`). - :param utctz: git utc timezone string, e.g. +0200 + :param utctz: + git utc timezone string, e.g. +0200 """ int_utctz = int(utctz) seconds = (abs(int_utctz) // 100) * 3600 + (abs(int_utctz) % 100) * 60 @@ -148,9 +155,11 @@ def utctz_to_altz(utctz: str) -> int: def altz_to_utctz_str(altz: float) -> str: - """Convert a timezone offset west of UTC in seconds into a Git timezone offset string. + """Convert a timezone offset west of UTC in seconds into a Git timezone offset + string. - :param altz: Timezone offset in seconds west of UTC + :param altz: + Timezone offset in seconds west of UTC. """ hours = abs(altz) // 3600 minutes = (abs(altz) % 3600) // 60 @@ -160,9 +169,11 @@ def altz_to_utctz_str(altz: float) -> str: def verify_utctz(offset: str) -> str: """ - :raise ValueError: If offset is incorrect + :raise ValueError: + If `offset` is incorrect. - :return: offset + :return: + `offset` """ fmt_exc = ValueError("Invalid timezone offset format: %s" % offset) if len(offset) != 5: @@ -197,7 +208,8 @@ def dst(self, dt: Union[datetime, None]) -> timedelta: def from_timestamp(timestamp: float, tz_offset: float) -> datetime: - """Convert a timestamp + tz_offset into an aware datetime instance.""" + """Convert a `timestamp` + `tz_offset` into an aware :class:`~datetime.datetime` + instance.""" utc_dt = datetime.fromtimestamp(timestamp, utc) try: local_dt = utc_dt.astimezone(tzoffset(tz_offset)) @@ -207,20 +219,22 @@ def from_timestamp(timestamp: float, tz_offset: float) -> datetime: def parse_date(string_date: Union[str, datetime]) -> Tuple[int, int]: - """ - Parse the given date as one of the following: + """Parse the given date as one of the following: * Aware datetime instance * Git internal format: timestamp offset - * RFC 2822: Thu, 07 Apr 2005 22:13:13 +0200. - * ISO 8601 2005-04-07T22:13:13 + * RFC 2822: ``Thu, 07 Apr 2005 22:13:13 +0200`` + * ISO 8601: ``2005-04-07T22:13:13`` The T can be a space as well. - :return: Tuple(int(timestamp_UTC), int(offset)), both in seconds since epoch + :return: + Tuple(int(timestamp_UTC), int(offset)), both in seconds since epoch - :raise ValueError: If the format could not be understood + :raise ValueError: + If the format could not be understood. - :note: Date can also be YYYY.MM.DD, MM/DD/YYYY and DD.MM.YYYY. + :note: + Date can also be ``YYYY.MM.DD``, ``MM/DD/YYYY`` and ``DD.MM.YYYY``. """ if isinstance(string_date, datetime): if string_date.tzinfo: @@ -314,7 +328,8 @@ def parse_actor_and_date(line: str) -> Tuple[Actor, int, int]: author Tom Preston-Werner 1191999972 -0700 - :return: [Actor, int_seconds_since_epoch, int_timezone_offset] + :return: + [Actor, int_seconds_since_epoch, int_timezone_offset] """ actor, epoch, offset = "", "0", "0" m = _re_actor_epoch.search(line) @@ -336,8 +351,8 @@ class ProcessStreamAdapter: """Class wiring all calls to the contained Process instance. Use this type to hide the underlying process to provide access only to a specified - stream. The process is usually wrapped into an AutoInterrupt class to kill - it if the instance goes out of scope. + stream. The process is usually wrapped into an :class:`~git.cmd.Git.AutoInterrupt` + class to kill it if the instance goes out of scope. """ __slots__ = ("_proc", "_stream") @@ -352,14 +367,18 @@ def __getattr__(self, attr: str) -> Any: @runtime_checkable class Traversable(Protocol): - """Simple interface to perform depth-first or breadth-first traversals - in one direction. + """Simple interface to perform depth-first or breadth-first traversals in one + direction. Subclasses only need to implement one function. - Instances of the Subclass must be hashable. + Instances of the subclass must be hashable. + + Defined subclasses: - Defined subclasses = [Commit, Tree, SubModule] + * :class:`Commit ` + * :class:`Tree ` + * :class:`Submodule ` """ __slots__ = () @@ -368,7 +387,7 @@ class Traversable(Protocol): @abstractmethod def _get_intermediate_items(cls, item: Any) -> Sequence["Traversable"]: """ - Returns: + :return: Tuple of items connected to the given item. Must be implemented in subclass. @@ -399,20 +418,23 @@ def _list_traverse( ) -> IterableList[Union["Commit", "Submodule", "Tree", "Blob"]]: """Traverse self and collect all items found. - :return: IterableList with the results of the traversal as produced by - :meth:`traverse`:: + :return: :class:`~git.util.IterableList` with the results of the traversal as + produced by :meth:`traverse`:: - Commit -> IterableList['Commit'] - Submodule -> IterableList['Submodule'] - Tree -> IterableList[Union['Submodule', 'Tree', 'Blob']] + Commit -> IterableList["Commit"] + Submodule -> IterableList["Submodule"] + Tree -> IterableList[Union["Submodule", "Tree", "Blob"]] """ # Commit and Submodule have id.__attribute__ as IterableObj. # Tree has id.__attribute__ inherited from IndexObject. if isinstance(self, Has_id_attribute): id = self._id_attribute_ else: - id = "" # Shouldn't reach here, unless Traversable subclass created with no _id_attribute_. - # Could add _id_attribute_ to Traversable, or make all Traversable also Iterable? + # Shouldn't reach here, unless Traversable subclass created with no + # _id_attribute_. + id = "" + # Could add _id_attribute_ to Traversable, or make all Traversable also + # Iterable? if not as_edge: out: IterableList[Union["Commit", "Submodule", "Tree", "Blob"]] = IterableList(id) @@ -453,46 +475,49 @@ def _traverse( ) -> Union[Iterator[Union["Traversable", "Blob"]], Iterator[TraversedTup]]: """Iterator yielding items found when traversing self. - :param predicate: f(i,d) returns False if item i at depth d should not be - included in the result. + :param predicate: + A function ``f(i,d)`` that returns False if item i at depth ``d`` should not + be included in the result. :param prune: - f(i,d) return True if the search should stop at item i at depth d. Item i - will not be returned. + A function ``f(i,d)`` that returns True if the search should stop at item + ``i`` at depth ``d``. Item ``i`` will not be returned. :param depth: Defines at which level the iteration should not go deeper if -1, there is no limit if 0, you would effectively only get self, the root of the iteration - i.e. if 1, you would only get the first level of predecessors/successors + i.e. if 1, you would only get the first level of predecessors/successors. :param branch_first: - if True, items will be returned branch first, otherwise depth first + If True, items will be returned branch first, otherwise depth first. :param visit_once: - if True, items will only be returned once, although they might be + If True, items will only be returned once, although they might be encountered several times. Loops are prevented that way. :param ignore_self: - if True, self will be ignored and automatically pruned from the result. - Otherwise it will be the first item to be returned. If as_edge is True, the + If True, self will be ignored and automatically pruned from the result. + Otherwise it will be the first item to be returned. If `as_edge` is True, the source of the first edge is None :param as_edge: - if True, return a pair of items, first being the source, second the + If True, return a pair of items, first being the source, second the destination, i.e. tuple(src, dest) with the edge spanning from source to - destination + destination. - :return: Iterator yielding items found when traversing self:: + :return: + Iterator yielding items found when traversing self:: - Commit -> Iterator[Union[Commit, Tuple[Commit, Commit]] - Submodule -> Iterator[Submodule, Tuple[Submodule, Submodule]] - Tree -> Iterator[Union[Blob, Tree, Submodule, - Tuple[Union[Submodule, Tree], Union[Blob, Tree, Submodule]]] + Commit -> Iterator[Union[Commit, Tuple[Commit, Commit]] Submodule -> + Iterator[Submodule, Tuple[Submodule, Submodule]] Tree -> + Iterator[Union[Blob, Tree, Submodule, + Tuple[Union[Submodule, Tree], Union[Blob, Tree, + Submodule]]] - ignore_self=True is_edge=True -> Iterator[item] - ignore_self=True is_edge=False --> Iterator[item] - ignore_self=False is_edge=True -> Iterator[item] | Iterator[Tuple[src, item]] - ignore_self=False is_edge=False -> Iterator[Tuple[src, item]] + ignore_self=True is_edge=True -> Iterator[item] ignore_self=True + is_edge=False --> Iterator[item] ignore_self=False is_edge=True -> + Iterator[item] | Iterator[Tuple[src, item]] ignore_self=False + is_edge=False -> Iterator[Tuple[src, item]] """ visited = set() @@ -526,7 +551,9 @@ def addToStack( visited.add(item) rval: Union[TraversedTup, "Traversable", "Blob"] - if as_edge: # If as_edge return (src, item) unless rrc is None (e.g. for first item). + if as_edge: + # If as_edge return (src, item) unless rrc is None + # (e.g. for first item). rval = (src, item) else: rval = item @@ -549,7 +576,8 @@ def addToStack( @runtime_checkable class Serializable(Protocol): - """Defines methods to serialize and deserialize objects from and into a data stream.""" + """Defines methods to serialize and deserialize objects from and into a data + stream.""" __slots__ = () @@ -557,11 +585,14 @@ class Serializable(Protocol): def _serialize(self, stream: "BytesIO") -> "Serializable": """Serialize the data of this object into the given data stream. - :note: A serialized object would :meth:`_deserialize` into the same object. + :note: + A serialized object would :meth:`_deserialize` into the same object. - :param stream: a file-like object + :param stream: + A file-like object. - :return: self + :return: + self """ raise NotImplementedError("To be implemented in subclass") @@ -569,9 +600,11 @@ def _serialize(self, stream: "BytesIO") -> "Serializable": def _deserialize(self, stream: "BytesIO") -> "Serializable": """Deserialize all information regarding this object from the stream. - :param stream: a file-like object + :param stream: + A file-like object. - :return: self + :return: + self """ raise NotImplementedError("To be implemented in subclass") From 37011bf873b93ad38ab184b3885df66072692a12 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 13:25:05 -0500 Subject: [PATCH 0683/1392] Fix backslash formatting in git.util docstrings As in 5219489. --- git/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/util.py b/git/util.py index 30b78f7b2..4126c6c73 100644 --- a/git/util.py +++ b/git/util.py @@ -260,7 +260,7 @@ def stream_copy(source: BinaryIO, destination: BinaryIO, chunk_size: int = 512 * def join_path(a: PathLike, *p: PathLike) -> PathLike: R"""Join path tokens together similar to osp.join, but always use - '/' instead of possibly '\' on Windows.""" + ``/`` instead of possibly ``\`` on Windows.""" path = str(a) for b in p: b = str(b) @@ -300,7 +300,7 @@ def join_path_native(a: PathLike, *p: PathLike) -> PathLike: R"""Like join_path, but makes sure an OS native path is returned. This is only needed to play it safe on Windows and to ensure nice paths that only - use '\'. + use ``\``. """ return to_native_path(join_path(a, *p)) From d9fb2f4c8f59f6074493d780a4ba62e89fa9f5d6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 13:36:49 -0500 Subject: [PATCH 0684/1392] Further git.util docstring revisions That I had missed in 1cd73ba. --- git/util.py | 52 +++++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/git/util.py b/git/util.py index 4126c6c73..72861ef84 100644 --- a/git/util.py +++ b/git/util.py @@ -201,9 +201,9 @@ def patch_env(name: str, value: str) -> Generator[None, None, None]: def rmtree(path: PathLike) -> None: """Remove the given directory tree recursively. - :note: We use :func:`shutil.rmtree` but adjust its behaviour to see whether files - that couldn't be deleted are read-only. Windows will not remove them in that - case. + :note: + We use :func:`shutil.rmtree` but adjust its behaviour to see whether files that + couldn't be deleted are read-only. Windows will not remove them in that case. """ def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: @@ -241,8 +241,8 @@ def rmfile(path: PathLike) -> None: def stream_copy(source: BinaryIO, destination: BinaryIO, chunk_size: int = 512 * 1024) -> int: - """Copy all data from the source stream into the destination stream in chunks - of size chunk_size. + """Copy all data from the `source` stream into the `destination` stream in chunks + of size `chunk_size`. :return: Number of bytes written @@ -259,8 +259,8 @@ def stream_copy(source: BinaryIO, destination: BinaryIO, chunk_size: int = 512 * def join_path(a: PathLike, *p: PathLike) -> PathLike: - R"""Join path tokens together similar to osp.join, but always use - ``/`` instead of possibly ``\`` on Windows.""" + R"""Join path tokens together similar to osp.join, but always use ``/`` instead of + possibly ``\`` on Windows.""" path = str(a) for b in p: b = str(b) @@ -297,7 +297,7 @@ def to_native_path_linux(path: PathLike) -> str: def join_path_native(a: PathLike, *p: PathLike) -> PathLike: - R"""Like join_path, but makes sure an OS native path is returned. + R"""Like :func:`join_path`, but makes sure an OS native path is returned. This is only needed to play it safe on Windows and to ensure nice paths that only use ``\``. @@ -308,10 +308,12 @@ def join_path_native(a: PathLike, *p: PathLike) -> PathLike: def assure_directory_exists(path: PathLike, is_file: bool = False) -> bool: """Make sure that the directory pointed to by path exists. - :param is_file: If True, ``path`` is assumed to be a file and handled correctly. + :param is_file: + If True, `path` is assumed to be a file and handled correctly. Otherwise it must be a directory. - :return: True if the directory was created, False if it already existed. + :return: + True if the directory was created, False if it already existed. """ if is_file: path = osp.dirname(path) @@ -339,7 +341,8 @@ def py_where(program: str, path: Optional[PathLike] = None) -> List[str]: :func:`is_cygwin_git`. When a search following all shell rules is needed, :func:`shutil.which` can be used instead. - :note: Neither this function nor :func:`shutil.which` will predict the effect of an + :note: + Neither this function nor :func:`shutil.which` will predict the effect of an executable search on a native Windows system due to a :class:`subprocess.Popen` call without ``shell=True``, because shell and non-shell executable search on Windows differ considerably. @@ -550,8 +553,7 @@ def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: class RemoteProgress: """Handler providing an interface to parse progress information emitted by ``git push`` and ``git fetch`` and to dispatch callbacks allowing subclasses to - react to the progress. - """ + react to the progress.""" _num_op_codes: int = 9 ( @@ -761,8 +763,8 @@ def update(self, *args: Any, **kwargs: Any) -> None: class Actor: """Actors hold information about a person acting on the repository. They - can be committers and authors or anything with a name and an email as - mentioned in the git log entries.""" + can be committers and authors or anything with a name and an email as mentioned in + the git log entries.""" # PRECOMPILED REGEX name_only_regex = re.compile(r"<(.*)>") @@ -802,7 +804,7 @@ def __repr__(self) -> str: @classmethod def _from_string(cls, string: str) -> "Actor": - """Create an Actor from a string. + """Create an :class:`Actor` from a string. :param string: The string, which is expected to be in regular git format:: @@ -868,10 +870,11 @@ def default_name() -> str: @classmethod def committer(cls, config_reader: Union[None, "GitConfigParser", "SectionConstraint"] = None) -> "Actor": """ - :return: Actor instance corresponding to the configured committer. It behaves - similar to the git implementation, such that the environment will override - configuration values of `config_reader`. If no value is set at all, it will - be generated. + :return: + :class:`Actor` instance corresponding to the configured committer. It + behaves similar to the git implementation, such that the environment will + override configuration values of `config_reader`. If no value is set at all, + it will be generated. :param config_reader: ConfigReader to use to retrieve the values from in case they are not set in @@ -887,8 +890,7 @@ def author(cls, config_reader: Union[None, "GitConfigParser", "SectionConstraint class Stats: - """ - Represents stat information as presented by git at the end of a merge. It is + """Represents stat information as presented by git at the end of a merge. It is created from the output of a diff operation. Example:: @@ -949,9 +951,9 @@ def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": class IndexFileSHA1Writer: - """Wrapper around a file-like object that remembers the SHA1 of - the data written to it. It will write a sha when the stream is closed - or if asked for explicitly using :meth:`write_sha`. + """Wrapper around a file-like object that remembers the SHA1 of the data written to + it. It will write a sha when the stream is closed or if asked for explicitly using + :meth:`write_sha`. Only useful to the index file. From d1d18c230b8853712cc44e0609b54cf55f09cafa Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 14:34:40 -0500 Subject: [PATCH 0685/1392] Revise docstrings within git.refs --- git/refs/head.py | 72 +++++++------- git/refs/log.py | 110 ++++++++++++++-------- git/refs/reference.py | 38 +++++--- git/refs/remote.py | 16 ++-- git/refs/symbolic.py | 213 +++++++++++++++++++++++++----------------- git/refs/tag.py | 20 ++-- 6 files changed, 283 insertions(+), 186 deletions(-) diff --git a/git/refs/head.py b/git/refs/head.py index ebc71eb96..a051748ba 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -52,8 +52,9 @@ def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME): def orig_head(self) -> SymbolicReference: """ - :return: SymbolicReference pointing at the ORIG_HEAD, which is maintained - to contain the previous value of HEAD. + :return: + :class:`~git.refs.symbolic.SymbolicReference` pointing at the ORIG_HEAD, + which is maintained to contain the previous value of HEAD. """ return SymbolicReference(self.repo, self._ORIG_HEAD_NAME) @@ -66,16 +67,16 @@ def reset( **kwargs: Any, ) -> "HEAD": """Reset our HEAD to the given commit optionally synchronizing - the index and working tree. The reference we refer to will be set to - commit as well. + the index and working tree. The reference we refer to will be set to commit as + well. :param commit: - Commit object, Reference Object or string identifying a revision we - should reset HEAD to. + :class:`~git.objects.commit.Commit`, :class:`~git.refs.reference.Reference`, + or string identifying a revision we should reset HEAD to. :param index: - If True, the index will be set to match the given commit. Otherwise - it will not be touched. + If True, the index will be set to match the given commit. + Otherwise it will not be touched. :param working_tree: If True, the working tree will be forcefully adjusted to match the given @@ -87,7 +88,7 @@ def reset( that are to be reset. This allows to partially reset individual files. :param kwargs: - Additional arguments passed to git-reset. + Additional arguments passed to ``git reset``. :return: self """ @@ -123,8 +124,8 @@ def reset( class Head(Reference): - """A Head is a named reference to a Commit. Every Head instance contains a name - and a Commit object. + """A Head is a named reference to a :class:`~git.objects.commit.Commit`. Every Head + instance contains a name and a :class:`~git.objects.commit.Commit` object. Examples:: @@ -150,9 +151,8 @@ def delete(cls, repo: "Repo", *heads: "Union[Head, str]", force: bool = False, * """Delete the given heads. :param force: - If True, the heads will be deleted even if they are not yet merged into - the main development stream. - Default False + If True, the heads will be deleted even if they are not yet merged into the + main development stream. Default False. """ flag = "-d" if force: @@ -163,9 +163,11 @@ def set_tracking_branch(self, remote_reference: Union["RemoteReference", None]) """Configure this branch to track the given remote reference. This will alter this branch's configuration accordingly. - :param remote_reference: The remote reference to track or None to untrack - any references. - :return: self + :param remote_reference: + The remote reference to track or None to untrack any references. + + :return: + self """ from .remote import RemoteReference @@ -190,8 +192,10 @@ def set_tracking_branch(self, remote_reference: Union["RemoteReference", None]) def tracking_branch(self) -> Union["RemoteReference", None]: """ - :return: The remote_reference we are tracking, or None if we are - not a tracking branch.""" + :return: + The remote reference we are tracking, or None if we are not a tracking + branch. + """ from .remote import RemoteReference reader = self.config_reader() @@ -211,16 +215,18 @@ def rename(self, new_path: PathLike, force: bool = False) -> "Head": """Rename self to a new path. :param new_path: - Either a simple name or a path, i.e. new_name or features/new_name. - The prefix refs/heads is implied. + Either a simple name or a path, e.g. ``new_name`` or ``features/new_name``. + The prefix ``refs/heads`` is implied. :param force: If True, the rename will succeed even if a head with the target name already exists. - :return: self + :return: + self - :note: Respects the ref log as git commands are used. + :note: + Respects the ref log as git commands are used. """ flag = "-m" if force: @@ -247,15 +253,15 @@ def checkout(self, force: bool = False, **kwargs: Any) -> Union["HEAD", "Head"]: ``b="new_branch"`` to create a new branch at the given spot. :return: - The active branch after the checkout operation, usually self unless - a new branch has been created. + The active branch after the checkout operation, usually self unless a new + branch has been created. If there is no active branch, as the HEAD is now detached, the HEAD reference will be returned instead. :note: - By default it is only allowed to checkout heads - everything else - will leave the HEAD detached which is allowed and possible, but remains - a special state that some tools might not be able to handle. + By default it is only allowed to checkout heads - everything else will leave + the HEAD detached which is allowed and possible, but remains a special state + that some tools might not be able to handle. """ kwargs["f"] = force if kwargs["f"] is False: @@ -279,15 +285,17 @@ def _config_parser(self, read_only: bool) -> SectionConstraint[GitConfigParser]: def config_reader(self) -> SectionConstraint[GitConfigParser]: """ - :return: A configuration parser instance constrained to only read - this instance's values. + :return: + A configuration parser instance constrained to only read this instance's + values. """ return self._config_parser(read_only=True) def config_writer(self) -> SectionConstraint[GitConfigParser]: """ - :return: A configuration writer instance with read-and write access - to options of this head. + :return: + A configuration writer instance with read-and write access to options of + this head. """ return self._config_parser(read_only=False) diff --git a/git/refs/log.py b/git/refs/log.py index e45798d8a..260f2fff5 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -44,6 +44,7 @@ class RefLogEntry(Tuple[str, str, Actor, Tuple[int, int], str]): """Named tuple allowing easy access to the revlog data fields.""" _re_hexsha_only = re.compile(r"^[0-9A-Fa-f]{40}$") + __slots__ = () def __repr__(self) -> str: @@ -81,7 +82,7 @@ def actor(self) -> Actor: @property def time(self) -> Tuple[int, int]: - """time as tuple: + """Time as tuple: * [0] = ``int(time)`` * [1] = ``int(timezone_offset)`` in :attr:`time.altzone` format @@ -113,9 +114,11 @@ def new( def from_line(cls, line: bytes) -> "RefLogEntry": """:return: New RefLogEntry instance from the given revlog line. - :param line: Line bytes without trailing newline + :param line: + Line bytes without trailing newline - :raise ValueError: If `line` could not be parsed + :raise ValueError: + If `line` could not be parsed. """ line_str = line.decode(defenc) fields = line_str.split("\t", 1) @@ -147,9 +150,9 @@ def from_line(cls, line: bytes) -> "RefLogEntry": class RefLog(List[RefLogEntry], Serializable): - """A reflog contains RefLogEntrys, each of which defines a certain state - of the head in question. Custom query methods allow to retrieve log entries - by date or by other criteria. + R"""A reflog contains :class:`RefLogEntry`\s, each of which defines a certain state + of the head in question. Custom query methods allow to retrieve log entries by date + or by other criteria. Reflog entries are ordered. The first added entry is first in the list. The last entry, i.e. the last change of the head or reference, is last in the list. @@ -163,8 +166,8 @@ def __new__(cls, filepath: Union[PathLike, None] = None) -> "RefLog": def __init__(self, filepath: Union[PathLike, None] = None): """Initialize this instance with an optional filepath, from which we will - initialize our data. The path is also used to write changes back using - the write() method.""" + initialize our data. The path is also used to write changes back using the + :meth:`write` method.""" self._path = filepath if filepath is not None: self._read_from_file() @@ -189,31 +192,40 @@ def _read_from_file(self) -> None: @classmethod def from_file(cls, filepath: PathLike) -> "RefLog": """ - :return: A new RefLog instance containing all entries from the reflog - at the given filepath - :param filepath: Path to reflog - :raise ValueError: If the file could not be read or was corrupted in some way + :return: + A new :class:`RefLog` instance containing all entries from the reflog at the + given `filepath`. + + :param filepath: + Path to reflog. + + :raise ValueError: + If the file could not be read or was corrupted in some way. """ return cls(filepath) @classmethod def path(cls, ref: "SymbolicReference") -> str: """ - :return: String to absolute path at which the reflog of the given ref - instance would be found. The path is not guaranteed to point to a valid - file though. - :param ref: SymbolicReference instance + :return: + String to absolute path at which the reflog of the given ref instance would + be found. The path is not guaranteed to point to a valid file though. + + :param ref: + :class:`~git.refs.symbolic.SymbolicReference` instance """ return osp.join(ref.repo.git_dir, "logs", to_native_path(ref.path)) @classmethod def iter_entries(cls, stream: Union[str, "BytesIO", mmap]) -> Iterator[RefLogEntry]: """ - :return: Iterator yielding RefLogEntry instances, one for each line read + :return: + Iterator yielding :class:`RefLogEntry` instances, one for each line read from the given stream. - :param stream: File-like object containing the revlog in its native format - or string instance pointing to a file to read. + :param stream: + File-like object containing the revlog in its native format or string + instance pointing to a file to read. """ new_entry = RefLogEntry.from_line if isinstance(stream, str): @@ -233,18 +245,23 @@ def iter_entries(cls, stream: Union[str, "BytesIO", mmap]) -> Iterator[RefLogEnt @classmethod def entry_at(cls, filepath: PathLike, index: int) -> "RefLogEntry": """ - :return: RefLogEntry at the given index. + :return: + :class:`RefLogEntry` at the given index. - :param filepath: Full path to the index file from which to read the entry. + :param filepath: + Full path to the index file from which to read the entry. - :param index: Python list compatible index, i.e. it may be negative to - specify an entry counted from the end of the list. + :param index: + Python list compatible index, i.e. it may be negative to specify an entry + counted from the end of the list. - :raise IndexError: If the entry didn't exist. + :raise IndexError: + If the entry didn't exist. - .. note:: This method is faster as it only parses the entry at index, skipping - all other lines. Nonetheless, the whole file has to be read if - the index is negative. + :note: + This method is faster as it only parses the entry at index, skipping all + other lines. Nonetheless, the whole file has to be read if the index is + negative. """ with open(filepath, "rb") as fp: if index < 0: @@ -264,7 +281,8 @@ def entry_at(cls, filepath: PathLike, index: int) -> "RefLogEntry": def to_file(self, filepath: PathLike) -> None: """Write the contents of the reflog instance to a file at the given filepath. - :param filepath: Path to file, parent directories are assumed to exist. + :param filepath: + Path to file. Parent directories are assumed to exist. """ lfd = LockedFD(filepath) assure_directory_exists(filepath, is_file=True) @@ -290,19 +308,32 @@ def append_entry( ) -> "RefLogEntry": """Append a new log entry to the revlog at filepath. - :param config_reader: Configuration reader of the repository - used to obtain - user information. May also be an Actor instance identifying the committer + :param config_reader: + Configuration reader of the repository - used to obtain user information. + May also be an :class:`~git.util.Actor` instance identifying the committer directly or None. - :param filepath: Full path to the log file. - :param oldbinsha: Binary sha of the previous commit. - :param newbinsha: Binary sha of the current commit. - :param message: Message describing the change to the reference. - :param write: If True, the changes will be written right away. Otherwise the - change will not be written. - :return: RefLogEntry objects which was appended to the log. + :param filepath: + Full path to the log file. + + :param oldbinsha: + Binary sha of the previous commit. + + :param newbinsha: + Binary sha of the current commit. + + :param message: + Message describing the change to the reference. + + :param write: + If True, the changes will be written right away. + Otherwise the change will not be written. + + :return: + :class:`RefLogEntry` objects which was appended to the log. - :note: As we are append-only, concurrent access is not a problem as we do not + :note: + As we are append-only, concurrent access is not a problem as we do not interfere with readers. """ @@ -340,7 +371,8 @@ def append_entry( def write(self) -> "RefLog": """Write this instance's data to the file we are originating from. - :return: self + :return: + self """ if self._path is None: raise ValueError("Instance was not initialized with a path, use to_file(...) instead") diff --git a/git/refs/reference.py b/git/refs/reference.py index 20d42472d..32547278f 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -25,7 +25,8 @@ def require_remote_ref_path(func: Callable[..., _T]) -> Callable[..., _T]: - """A decorator raising a TypeError if we are not a valid remote, based on the path.""" + """A decorator raising :class:`TypeError` if we are not a valid remote, based on the + path.""" def wrapper(self: T_References, *args: Any) -> _T: if not self.is_remote(): @@ -56,11 +57,16 @@ class Reference(SymbolicReference, LazyMixin, IterableObj): def __init__(self, repo: "Repo", path: PathLike, check_path: bool = True) -> None: """Initialize this instance. - :param repo: Our parent repository. - :param path: Path relative to the .git/ directory pointing to the ref in - question, e.g. ``refs/heads/master``. - :param check_path: If False, you can provide any path. Otherwise the path must - start with the default path prefix of this type. + :param repo: + Our parent repository. + + :param path: + Path relative to the ``.git/`` directory pointing to the ref in question, + e.g. ``refs/heads/master``. + + :param check_path: + If False, you can provide any path. + Otherwise the path must start with the default path prefix of this type. """ if check_path and not str(path).startswith(self._common_path_default + "/"): raise ValueError(f"Cannot instantiate {self.__class__.__name__!r} from path {path}") @@ -80,7 +86,8 @@ def set_object( ) -> "Reference": """Special version which checks if the head-log needs an update as well. - :return: self + :return: + self """ oldbinsha = None if logmsg is not None: @@ -115,7 +122,10 @@ def set_object( @property def name(self) -> str: - """:return: (shortest) Name of this reference - it may contain path components""" + """ + :return: + (shortest) Name of this reference - it may contain path components + """ # The first two path tokens can be removed as they are # refs/heads or refs/tags or refs/remotes. tokens = self.path.split("/") @@ -144,8 +154,8 @@ def iter_items( def remote_name(self) -> str: """ :return: - Name of the remote we are a reference of, such as 'origin' for a reference - named 'origin/master'. + Name of the remote we are a reference of, such as ``origin`` for a reference + named ``origin/master``. """ tokens = self.path.split("/") # /refs/remotes// @@ -155,10 +165,12 @@ def remote_name(self) -> str: @require_remote_ref_path def remote_head(self) -> str: """ - :return: Name of the remote head itself, e.g. master. + :return: + Name of the remote head itself, e.g. ``master``. - :note: The returned name is usually not qualified enough to uniquely identify - a branch. + :note: + The returned name is usually not qualified enough to uniquely identify a + branch. """ tokens = self.path.split("/") return "/".join(tokens[3:]) diff --git a/git/refs/remote.py b/git/refs/remote.py index 59d02a755..c2c2c1aac 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -47,10 +47,10 @@ def iter_items( # super is Reference return super().iter_items(repo, common_path) - # The Head implementation of delete also accepts strs, but this - # implementation does not. mypy doesn't have a way of representing - # tightening the types of arguments in subclasses and recommends Any or - # "type: ignore". (See https://github.com/python/typing/issues/241) + # The Head implementation of delete also accepts strs, but this implementation does + # not. mypy doesn't have a way of representing tightening the types of arguments in + # subclasses and recommends Any or "type: ignore". + # (See: https://github.com/python/typing/issues/241) @classmethod def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: # type: ignore """Delete the given remote references. @@ -60,9 +60,9 @@ def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: should not narrow the signature. """ repo.git.branch("-d", "-r", *refs) - # The official deletion method will ignore remote symbolic refs - these - # are generally ignored in the refs/ folder. We don't though - # and delete remainders manually. + # The official deletion method will ignore remote symbolic refs - these are + # generally ignored in the refs/ folder. We don't though and delete remainders + # manually. for ref in refs: try: os.remove(os.path.join(repo.common_dir, ref.path)) @@ -76,5 +76,5 @@ def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: @classmethod def create(cls, *args: Any, **kwargs: Any) -> NoReturn: - """Raises TypeError. Defined so the create method is disabled.""" + """Raise :class:`TypeError`. Defined so the ``create`` method is disabled.""" raise TypeError("Cannot explicitly create remote references") diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 31f959ac2..6b9fd9ab7 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -100,8 +100,8 @@ def __hash__(self) -> int: def name(self) -> str: """ :return: - In case of symbolic references, the shortest assumable name - is the path itself. + In case of symbolic references, the shortest assumable name is the path + itself. """ return str(self.path) @@ -118,7 +118,8 @@ def _iter_packed_refs(cls, repo: "Repo") -> Iterator[Tuple[str, str]]: """Return an iterator yielding pairs of sha1/path pairs (as strings) for the corresponding refs. - :note: The packed refs file will be kept open as long as we iterate. + :note: + The packed refs file will be kept open as long as we iterate. """ try: with open(cls._get_packed_refs_path(repo), "rt", encoding="UTF-8") as fp: @@ -155,10 +156,12 @@ def _iter_packed_refs(cls, repo: "Repo") -> Iterator[Tuple[str, str]]: @classmethod def dereference_recursive(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> str: """ - :return: hexsha stored in the reference at the given ref_path, recursively dereferencing all - intermediate references as required + :return: + hexsha stored in the reference at the given `ref_path`, recursively + dereferencing all intermediate references as required - :param repo: The repository containing the reference at ref_path + :param repo: + The repository containing the reference at `ref_path`. """ while True: @@ -221,8 +224,9 @@ def _get_ref_info_helper( cls, repo: "Repo", ref_path: Union[PathLike, None] ) -> Union[Tuple[str, None], Tuple[None, str]]: """ - :return: (str(sha), str(target_ref_path)) if available, the sha the file at - rela_path points to, or None. + :return: + (str(sha), str(target_ref_path)) if available, the sha the file at rela_path + points to, or None. target_ref_path is the reference we point to, or None. """ @@ -276,8 +280,8 @@ def _get_ref_info(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> Union[T def _get_object(self) -> Commit_ish: """ :return: - The object our ref currently refers to. Refs can be cached, they will - always point to the actual object as it gets re-created on each query. + The object our ref currently refers to. Refs can be cached, they will always + point to the actual object as it gets re-created on each query. """ # We have to be dynamic here as we may be a tag which can point to anything. # Our path will be resolved to the hexsha which will be used accordingly. @@ -305,11 +309,15 @@ def set_commit( commit: Union[Commit, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "SymbolicReference": - """As set_object, but restricts the type of object to be a Commit. + """Like :meth:`set_object`, but restricts the type of object to be a + :class:`~git.objects.commit.Commit`. + + :raise ValueError: + If `commit` is not a :class:`~git.objects.commit.Commit` object or doesn't + point to a commit. - :raise ValueError: If commit is not a :class:`~git.objects.commit.Commit` object - or doesn't point to a commit - :return: self + :return: + self """ # Check the type - assume the best if it is a base-string. invalid_type = False @@ -339,18 +347,25 @@ def set_object( object: Union[Commit_ish, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "SymbolicReference": - """Set the object we point to, possibly dereference our symbolic reference first. - If the reference does not exist, it will be created. - - :param object: A refspec, a :class:`SymbolicReference` or an - :class:`~git.objects.base.Object` instance. - :class:`SymbolicReference` instances will be dereferenced beforehand to - obtain the object they point to. - :param logmsg: If not None, the message will be used in the reflog entry to be - written. Otherwise the reflog is not altered. - :note: Plain :class:`SymbolicReference` instances may not actually point to - objects by convention. - :return: self + """Set the object we point to, possibly dereference our symbolic reference + first. If the reference does not exist, it will be created. + + :param object: + A refspec, a :class:`SymbolicReference` or an + :class:`~git.objects.base.Object` instance. :class:`SymbolicReference` + instances will be dereferenced beforehand to obtain the object they point + to. + + :param logmsg: + If not None, the message will be used in the reflog entry to be written. + Otherwise the reflog is not altered. + + :note: + Plain :class:`SymbolicReference` instances may not actually point to objects + by convention. + + :return: + self """ if isinstance(object, SymbolicReference): object = object.object # @ReservedAssignment @@ -374,10 +389,13 @@ def set_object( def _get_reference(self) -> "SymbolicReference": """ - :return: Reference Object we point to + :return: + :class:`~git.refs.reference.Reference` object we point to - :raise TypeError: If this symbolic reference is detached, hence it doesn't point - to a reference, but to a commit""" + :raise TypeError: + If this symbolic reference is detached, hence it doesn't point to a + reference, but to a commit. + """ sha, target_ref_path = self._get_ref_info(self.repo, self.path) if target_ref_path is None: raise TypeError("%s is a detached symbolic reference as it points to %r" % (self, sha)) @@ -388,10 +406,13 @@ def set_reference( ref: Union[Commit_ish, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "SymbolicReference": - """Set ourselves to the given ref. It will stay a symbol if the ref is a Reference. - Otherwise an Object, given as Object instance or refspec, is assumed and if valid, - will be set which effectively detaches the reference if it was a purely - symbolic one. + """Set ourselves to the given `ref`. + + It will stay a symbol if the ref is a :class:`~git.refs.reference.Reference`. + + Otherwise an Object, given as :class:`~git.objects.base.Object` instance or + refspec, is assumed and if valid, will be set which effectively detaches the + reference if it was a purely symbolic one. :param ref: A :class:`SymbolicReference` instance, an :class:`~git.objects.base.Object` @@ -399,15 +420,18 @@ def set_reference( :class:`SymbolicReference` instance, we will point to it. Everything else is dereferenced to obtain the actual object. - :param logmsg: If set to a string, the message will be used in the reflog. + :param logmsg: + If set to a string, the message will be used in the reflog. Otherwise, a reflog entry is not written for the changed reference. The previous commit of the entry will be the commit we point to now. See also: :meth:`log_append` - :return: self + :return: + self - :note: This symbolic reference will not be dereferenced. For that, see + :note: + This symbolic reference will not be dereferenced. For that, see :meth:`set_object`. """ write_value = None @@ -467,8 +491,8 @@ def set_reference( def is_valid(self) -> bool: """ :return: - True if the reference is valid, hence it can be read and points to - a valid object or reference. + True if the reference is valid, hence it can be read and points to a valid + object or reference. """ try: self.object @@ -492,11 +516,13 @@ def is_detached(self) -> bool: def log(self) -> "RefLog": """ - :return: RefLog for this reference. Its last entry reflects the latest change - applied to this reference. + :return: + :class:`~git.refs.log.RefLog` for this reference. + Its last entry reflects the latest change applied to this reference. - .. note:: As the log is parsed every time, its recommended to cache it for use - instead of calling this method repeatedly. It should be considered read-only. + :note: + As the log is parsed every time, its recommended to cache it for use instead + of calling this method repeatedly. It should be considered read-only. """ return RefLog.from_file(RefLog.path(self)) @@ -508,11 +534,17 @@ def log_append( ) -> "RefLogEntry": """Append a logentry to the logfile of this ref. - :param oldbinsha: Binary sha this ref used to point to. - :param message: A message describing the change. - :param newbinsha: The sha the ref points to now. If None, our current commit sha - will be used. - :return: The added :class:`~git.refs.log.RefLogEntry` instance. + :param oldbinsha: + Binary sha this ref used to point to. + + :param message: + A message describing the change. + + :param newbinsha: + The sha the ref points to now. If None, our current commit sha will be used. + + :return: + The added :class:`~git.refs.log.RefLogEntry` instance. """ # NOTE: We use the committer of the currently active commit - this should be # correct to allow overriding the committer on a per-commit level. @@ -532,20 +564,24 @@ def log_append( def log_entry(self, index: int) -> "RefLogEntry": """ - :return: RefLogEntry at the given index + :return: + RefLogEntry at the given index - :param index: Python list compatible positive or negative index + :param index: + Python list compatible positive or negative index. - .. note:: This method must read part of the reflog during execution, hence - it should be used sparingly, or only if you need just one index. - In that case, it will be faster than the :meth:`log` method. + :note: + This method must read part of the reflog during execution, hence it should + be used sparingly, or only if you need just one index. In that case, it will + be faster than the :meth:`log` method. """ return RefLog.entry_at(RefLog.path(self), index) @classmethod def to_full_path(cls, path: Union[PathLike, "SymbolicReference"]) -> PathLike: """ - :return: string with a full repository-relative path which can be used to initialize + :return: + String with a full repository-relative path which can be used to initialize a Reference instance, for instance by using :meth:`Reference.from_path `. """ @@ -566,8 +602,8 @@ def delete(cls, repo: "Repo", path: PathLike) -> None: Repository to delete the reference from. :param path: - Short or full path pointing to the reference, e.g. ``refs/myreference`` - or just ``myreference``, hence ``refs/`` is implied. + Short or full path pointing to the reference, e.g. ``refs/myreference`` or + just ``myreference``, hence ``refs/`` is implied. Alternatively the symbolic reference to be deleted. """ full_ref_path = cls.to_full_path(path) @@ -586,10 +622,10 @@ def delete(cls, repo: "Repo", path: PathLike) -> None: line = line_bytes.decode(defenc) _, _, line_ref = line.partition(" ") line_ref = line_ref.strip() - # Keep line if it is a comment or if the ref to delete is not - # in the line. - # If we deleted the last line and this one is a tag-reference object, - # we drop it as well. + # Keep line if it is a comment or if the ref to delete is not in + # the line. + # If we deleted the last line and this one is a tag-reference + # object, we drop it as well. if (line.startswith("#") or full_ref_path != line_ref) and ( not dropped_last_line or dropped_last_line and not line.startswith("^") ): @@ -604,8 +640,8 @@ def delete(cls, repo: "Repo", path: PathLike) -> None: # Write the new lines. if made_change: - # Binary writing is required, otherwise Windows will - # open the file in text mode and change LF to CRLF! + # Binary writing is required, otherwise Windows will open the file + # in text mode and change LF to CRLF! with open(pack_file_path, "wb") as fd: fd.writelines(line.encode(defenc) for line in new_lines) @@ -630,10 +666,9 @@ def _create( ) -> T_References: """Internal method used to create a new symbolic reference. - If `resolve` is False, the reference will be taken as is, creating - a proper symbolic reference. Otherwise it will be resolved to the - corresponding object and a detached symbolic reference will be created - instead. + If `resolve` is False, the reference will be taken as is, creating a proper + symbolic reference. Otherwise it will be resolved to the corresponding object + and a detached symbolic reference will be created instead. """ git_dir = _git_dir(repo, path) full_ref_path = cls.to_full_path(path) @@ -679,28 +714,30 @@ def create( Repository to create the reference in. :param path: - Full path at which the new symbolic reference is supposed to be - created at, e.g. ``NEW_HEAD`` or ``symrefs/my_new_symref``. + Full path at which the new symbolic reference is supposed to be created at, + e.g. ``NEW_HEAD`` or ``symrefs/my_new_symref``. :param reference: The reference which the new symbolic reference should point to. If it is a commit-ish, the symbolic ref will be detached. :param force: - If True, force creation even if a symbolic reference with that name already exists. - Raise :class:`OSError` otherwise. + If True, force creation even if a symbolic reference with that name already + exists. Raise :class:`OSError` otherwise. :param logmsg: - If not None, the message to append to the reflog. Otherwise no reflog - entry is written. + If not None, the message to append to the reflog. + If None, no reflog entry is written. - :return: Newly created symbolic Reference + :return: + Newly created symbolic reference :raise OSError: - If a (Symbolic)Reference with the same name but different contents - already exists. + If a (Symbolic)Reference with the same name but different contents already + exists. - :note: This does not alter the current HEAD, index or working tree. + :note: + This does not alter the current HEAD, index or working tree. """ return cls._create(repo, path, cls._resolve_ref_on_create, reference, force, logmsg) @@ -708,17 +745,20 @@ def rename(self, new_path: PathLike, force: bool = False) -> "SymbolicReference" """Rename self to a new path. :param new_path: - Either a simple name or a full path, e.g. ``new_name`` or ``features/new_name``. + Either a simple name or a full path, e.g. ``new_name`` or + ``features/new_name``. The prefix ``refs/`` is implied for references and will be set as needed. In case this is a symbolic ref, there is no implied prefix. :param force: - If True, the rename will succeed even if a head with the target name - already exists. It will be overwritten in that case. + If True, the rename will succeed even if a head with the target name already + exists. It will be overwritten in that case. - :return: self + :return: + self - :raise OSError: If a file at path but with different contents already exists. + :raise OSError: + If a file at path but with different contents already exists. """ new_path = self.to_full_path(new_path) if self.path == new_path: @@ -801,7 +841,8 @@ def iter_items( ) -> Iterator[T_References]: """Find all refs in the repository. - :param repo: is the Repo + :param repo: + The :class:`~git.repo.base.Repo`. :param common_path: Optional keyword argument to the path which is to be shared by all returned @@ -821,13 +862,13 @@ def iter_items( @classmethod def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_References: - """ - Make a symbolic reference from a path. + """Make a symbolic reference from a path. - :param path: Full ``.git``-directory-relative path name to the Reference to - instantiate. + :param path: + Full ``.git``-directory-relative path name to the Reference to instantiate. - :note: Use :meth:`to_full_path` if you only have a partial path of a known + :note: + Use :meth:`to_full_path` if you only have a partial path of a known Reference type. :return: diff --git a/git/refs/tag.py b/git/refs/tag.py index a59a51337..7edd1d12a 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -43,7 +43,8 @@ class TagReference(Reference): def commit(self) -> "Commit": # type: ignore[override] # LazyMixin has unrelated commit method """:return: Commit object the tag ref points to - :raise ValueError: If the tag points to a tree or blob + :raise ValueError: + If the tag points to a tree or blob. """ obj = self.object while obj.type != "commit": @@ -63,8 +64,9 @@ def commit(self) -> "Commit": # type: ignore[override] # LazyMixin has unrelat @property def tag(self) -> Union["TagObject", None]: """ - :return: Tag object this tag ref points to or None in case - we are a lightweight tag""" + :return: + Tag object this tag ref points to or None in case we are a lightweight tag + """ obj = self.object if obj.type == "tag": return obj @@ -97,21 +99,23 @@ def create( :param logmsg: If not None, the message will be used in your tag object. This will also - create an additional tag object that allows to obtain that information, e.g.:: + create an additional tag object that allows to obtain that information, + e.g.:: tagref.tag.message :param message: - Synonym for the `logmsg` parameter. - Included for backwards compatibility. `logmsg` takes precedence if both are passed. + Synonym for the `logmsg` parameter. Included for backwards compatibility. + `logmsg` takes precedence if both are passed. :param force: If True, force creation of a tag even though that tag already exists. :param kwargs: - Additional keyword arguments to be passed to git-tag. + Additional keyword arguments to be passed to ``git tag``. - :return: A new TagReference. + :return: + A new :class:`TagReference`. """ if "ref" in kwargs and kwargs["ref"]: reference = kwargs["ref"] From 5d6c86a47cb72d7ca80d122bc0bb899eb0637b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Can=20Ta=C5=9Fl=C4=B1=C3=A7ukur?= Date: Wed, 28 Feb 2024 00:24:59 +0300 Subject: [PATCH 0686/1392] test: :white_check_mark: Added test for external diff engine and removed comment --- git/diff.py | 1 - test/test_diff.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/git/diff.py b/git/diff.py index ae85e77ae..6d4474d3e 100644 --- a/git/diff.py +++ b/git/diff.py @@ -155,7 +155,6 @@ def diff( if create_patch: args.append("-p") - # we expect the default diff driver args.append("--no-ext-diff") else: args.append("--raw") diff --git a/test/test_diff.py b/test/test_diff.py index 87f92f5d1..e3d0b8e5c 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -473,3 +473,45 @@ def test_rename_override(self, rw_dir): self.assertEqual(True, diff.renamed_file) self.assertEqual("file_a.txt", diff.rename_from) self.assertEqual("file_b.txt", diff.rename_to) + + @with_rw_directory + def test_diff_patch_with_external_engine(self, rw_dir): + repo = Repo.init(rw_dir) + gitignore = osp.join(rw_dir, ".gitignore") + + # First commit + with open(gitignore, "w") as f: + f.write("first_line\n") + repo.git.add(".gitignore") + repo.index.commit("first commit") + + # Adding second line and committing + with open(gitignore, "a") as f: + f.write("second_line\n") + repo.git.add(".gitignore") + repo.index.commit("second commit") + + # Adding third line and staging + with open(gitignore, "a") as f: + f.write("third_line\n") + repo.git.add(".gitignore") + + # Adding fourth line + with open(gitignore, "a") as f: + f.write("fourth_line\n") + + # Set the external diff engine + with repo.config_writer(config_level="repository") as writer: + writer.set_value("diff", "external", "bogus_diff_engine") + + head_against_head = repo.head.commit.diff("HEAD^", create_patch=True) + self.assertEqual(len(head_against_head), 1) + head_against_index = repo.head.commit.diff(create_patch=True) + self.assertEqual(len(head_against_index), 1) + head_against_working_tree = repo.head.commit.diff(None, create_patch=True) + self.assertEqual(len(head_against_working_tree), 1) + + index_against_head = repo.index.diff("HEAD", create_patch=True) + self.assertEqual(len(index_against_head), 1) + index_against_working_tree = repo.index.diff(None, create_patch=True) + self.assertEqual(len(index_against_working_tree), 1) From 4f6736995cf9816773757b9a6808d6eaabd08fd4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 18:04:12 -0500 Subject: [PATCH 0687/1392] Fix backslashes in Repo.__init__ docstring The example code block included a Windows-style path string written with doubled backslashes, but the docstring itself was not a raw string literal, so these collapsed into single backslashes. This makes the whole docstring an R-string, which is sufficient to solve that problem (since the backslashes are in a code block, Sphinx does not itself treat them as metacharacters). In addition, this changes the Windows-style path to be an R-string rather than using doubled backslashes. This is just to improve clarity (and remind readers they can use an R-string for Windows paths), and does not affect correctness. --- git/repo/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index bf7c2cc0d..c840e8a30 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -168,7 +168,7 @@ def __init__( search_parent_directories: bool = False, expand_vars: bool = True, ) -> None: - """Create a new Repo instance. + R"""Create a new Repo instance. :param path: The path to either the root git directory or the bare git repo:: @@ -177,7 +177,7 @@ def __init__( repo = Repo("/Users/mtrier/Development/git-python.git") repo = Repo("~/Development/git-python.git") repo = Repo("$REPOSITORIES/Development/git-python.git") - repo = Repo("C:\\Users\\mtrier\\Development\\git-python\\.git") + repo = Repo(R"C:\Users\mtrier\Development\git-python\.git") - In *Cygwin*, path may be a ``cygdrive/...`` prefixed path. - If it evaluates to false, :envvar:`GIT_DIR` is used, and if this also From 0c8ca1a9bdc7852e0e8b86923765c8cf812154e6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 18:45:26 -0500 Subject: [PATCH 0688/1392] Fix Repo.iter_commits docstring about return type It had said it returned a list of Commit objects, but it returns an iterator of Commit objects. --- git/repo/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index c840e8a30..598921ca9 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -692,7 +692,7 @@ def iter_commits( paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any, ) -> Iterator[Commit]: - """A list of Commit objects representing the history of a given ref/commit. + """An iterator of Commit objects representing the history of a given ref/commit. :param rev: Revision specifier, see git-rev-parse for viable options. @@ -708,7 +708,7 @@ def iter_commits( :note: To receive only commits between two named revisions, use the ``"revA...revB"`` revision specifier. - :return: ``git.Commit[]`` + :return: Iterator of ``git.Commit`` """ if rev is None: rev = self.head.commit From b2b6f7c83cafd69ee683dfc2546a98c8699d0a6a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 19:54:09 -0500 Subject: [PATCH 0689/1392] Revise docstrings within git.repo --- git/repo/__init__.py | 2 +- git/repo/base.py | 479 ++++++++++++++++++++++++++----------------- git/repo/fun.py | 120 ++++++----- 3 files changed, 366 insertions(+), 235 deletions(-) diff --git a/git/repo/__init__.py b/git/repo/__init__.py index a63d77878..50a8c6f86 100644 --- a/git/repo/__init__.py +++ b/git/repo/__init__.py @@ -1,6 +1,6 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Initialize the Repo package.""" +"""Initialize the repo package.""" from .base import Repo as Repo # noqa: F401 diff --git a/git/repo/base.py b/git/repo/base.py index 598921ca9..afbc1d75d 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -103,26 +103,34 @@ class BlameEntry(NamedTuple): class Repo: """Represents a git repository and allows you to query references, - gather commit information, generate diffs, create and clone repositories query + gather commit information, generate diffs, create and clone repositories, and query the log. The following attributes are worth using: - 'working_dir' is the working directory of the git command, which is the working tree - directory if available or the .git directory in case of bare repositories + * :attr:`working_dir` is the working directory of the git command, which is the + working tree directory if available or the ``.git`` directory in case of bare + repositories. - 'working_tree_dir' is the working tree directory, but will return None - if we are a bare repository. + * :attr:`working_tree_dir` is the working tree directory, but will return None if we + are a bare repository. - 'git_dir' is the .git repository directory, which is always set. + * :attr:`git_dir` is the ``.git`` repository directory, which is always set. """ DAEMON_EXPORT_FILE = "git-daemon-export-ok" - git = cast("Git", None) # Must exist, or __del__ will fail in case we raise on `__init__()`. + # Must exist, or __del__ will fail in case we raise on `__init__()`. + git = cast("Git", None) + working_dir: PathLike + """The working directory of the git command.""" + _working_tree_dir: Optional[PathLike] = None + git_dir: PathLike + """The ``.git`` repository directory.""" + _common_dir: PathLike = "" # Precompiled regex @@ -168,7 +176,7 @@ def __init__( search_parent_directories: bool = False, expand_vars: bool = True, ) -> None: - R"""Create a new Repo instance. + R"""Create a new :class:`Repo` instance. :param path: The path to either the root git directory or the bare git repo:: @@ -179,14 +187,14 @@ def __init__( repo = Repo("$REPOSITORIES/Development/git-python.git") repo = Repo(R"C:\Users\mtrier\Development\git-python\.git") - - In *Cygwin*, path may be a ``cygdrive/...`` prefixed path. - - If it evaluates to false, :envvar:`GIT_DIR` is used, and if this also - evals to false, the current-directory is used. + - In *Cygwin*, `path` may be a ``cygdrive/...`` prefixed path. + - If `path` is None or an empty string, :envvar:`GIT_DIR` is used. If that + environment variable is absent or empty, the current directory is used. :param odbt: - Object DataBase type - a type which is constructed by providing - the directory containing the database objects, i.e. .git/objects. It will - be used to access all object data + Object DataBase type - a type which is constructed by providing the + directory containing the database objects, i.e. ``.git/objects``. It will be + used to access all object data :param search_parent_directories: If True, all parent directories will be searched for a valid repo as well. @@ -195,18 +203,20 @@ def __init__( GitPython, which is considered a bug though. :raise InvalidGitRepositoryError: + :raise NoSuchPathError: - :return: git.Repo + :return: + git.Repo """ epath = path or os.getenv("GIT_DIR") if not epath: epath = os.getcwd() if Git.is_cygwin(): - # Given how the tests are written, this seems more likely to catch - # Cygwin git used from Windows than Windows git used from Cygwin. - # Therefore changing to Cygwin-style paths is the relevant operation. + # Given how the tests are written, this seems more likely to catch Cygwin + # git used from Windows than Windows git used from Cygwin. Therefore + # changing to Cygwin-style paths is the relevant operation. epath = cygpath(epath) epath = epath or path or os.getcwd() @@ -223,25 +233,26 @@ def __init__( if not os.path.exists(epath): raise NoSuchPathError(epath) - ## Walk up the path to find the `.git` dir. - # + # Walk up the path to find the `.git` dir. curpath = epath git_dir = None while curpath: # ABOUT osp.NORMPATH - # It's important to normalize the paths, as submodules will otherwise initialize their - # repo instances with paths that depend on path-portions that will not exist after being - # removed. It's just cleaner. + # It's important to normalize the paths, as submodules will otherwise + # initialize their repo instances with paths that depend on path-portions + # that will not exist after being removed. It's just cleaner. if is_git_dir(curpath): git_dir = curpath # from man git-config : core.worktree - # Set the path to the root of the working tree. If GIT_COMMON_DIR environment - # variable is set, core.worktree is ignored and not used for determining the - # root of working tree. This can be overridden by the GIT_WORK_TREE environment - # variable. The value can be an absolute path or relative to the path to the .git - # directory, which is either specified by GIT_DIR, or automatically discovered. - # If GIT_DIR is specified but none of GIT_WORK_TREE and core.worktree is specified, - # the current working directory is regarded as the top level of your working tree. + # Set the path to the root of the working tree. If GIT_COMMON_DIR + # environment variable is set, core.worktree is ignored and not used for + # determining the root of working tree. This can be overridden by the + # GIT_WORK_TREE environment variable. The value can be an absolute path + # or relative to the path to the .git directory, which is either + # specified by GIT_DIR, or automatically discovered. If GIT_DIR is + # specified but none of GIT_WORK_TREE and core.worktree is specified, + # the current working directory is regarded as the top level of your + # working tree. self._working_tree_dir = os.path.dirname(git_dir) if os.environ.get("GIT_COMMON_DIR") is None: gitconf = self._config_reader("repository", git_dir) @@ -359,7 +370,8 @@ def _set_description(self, descr: str) -> None: @property def working_tree_dir(self) -> Optional[PathLike]: """ - :return: The working tree directory of our git repository. + :return: + The working tree directory of our git repository. If this is a bare repository, None is returned. """ return self._working_tree_dir @@ -367,8 +379,9 @@ def working_tree_dir(self) -> Optional[PathLike]: @property def common_dir(self) -> PathLike: """ - :return: The git dir that holds everything except possibly HEAD, - FETCH_HEAD, ORIG_HEAD, COMMIT_EDITMSG, index, and logs/. + :return: + The git dir that holds everything except possibly HEAD, FETCH_HEAD, + ORIG_HEAD, COMMIT_EDITMSG, index, and logs/. """ return self._common_dir or self.git_dir @@ -379,17 +392,21 @@ def bare(self) -> bool: @property def heads(self) -> "IterableList[Head]": - """A list of ``Head`` objects representing the branch heads in this repo. + """A list of :class:`~git.refs.head.Head` objects representing the branch heads + in this repo. - :return: ``git.IterableList(Head, ...)`` + :return: + ``git.IterableList(Head, ...)`` """ return Head.list_items(self) @property def references(self) -> "IterableList[Reference]": - """A list of Reference objects representing tags, heads and remote references. + """A list of :class:`~git.refs.reference.Reference` objects representing tags, + heads and remote references. - :return: ``git.IterableList(Reference, ...)`` + :return: + ``git.IterableList(Reference, ...)`` """ return Reference.list_items(self) @@ -402,10 +419,11 @@ def references(self) -> "IterableList[Reference]": @property def index(self) -> "IndexFile": """ - :return: A :class:`~git.index.base.IndexFile` representing this repository's - index. + :return: + A :class:`~git.index.base.IndexFile` representing this repository's index. - :note: This property can be expensive, as the returned + :note: + This property can be expensive, as the returned :class:`~git.index.base.IndexFile` will be reinitialized. It is recommended to reuse the object. """ @@ -413,21 +431,27 @@ def index(self) -> "IndexFile": @property def head(self) -> "HEAD": - """:return: HEAD Object pointing to the current head reference""" + """ + :return: + :class:`~git.refs.head.HEAD` object pointing to the current head reference + """ return HEAD(self, "HEAD") @property def remotes(self) -> "IterableList[Remote]": - """A list of Remote objects allowing to access and manipulate remotes. + """A list of :class:`~git.remote.Remote` objects allowing to access and + manipulate remotes. - :return: ``git.IterableList(Remote, ...)`` + :return: + ``git.IterableList(Remote, ...)`` """ return Remote.list_items(self) def remote(self, name: str = "origin") -> "Remote": - """:return: Remote with the specified name + """:return: The remote with the specified name - :raise ValueError: If no remote with such a name exists + :raise ValueError + If no remote with such a name exists. """ r = Remote(self, name) if not r.exists(): @@ -439,15 +463,17 @@ def remote(self, name: str = "origin") -> "Remote": @property def submodules(self) -> "IterableList[Submodule]": """ - :return: git.IterableList(Submodule, ...) of direct submodules - available from the current head + :return: + git.IterableList(Submodule, ...) of direct submodules available from the + current head """ return Submodule.list_items(self) def submodule(self, name: str) -> "Submodule": - """:return: Submodule with the given name + """:return: The submodule with the given name - :raise ValueError: If no such submodule exists + :raise ValueError: + If no such submodule exists. """ try: return self.submodules[name] @@ -458,10 +484,12 @@ def submodule(self, name: str) -> "Submodule": def create_submodule(self, *args: Any, **kwargs: Any) -> Submodule: """Create a new submodule. - :note: See the documentation of Submodule.add for a description of the - applicable parameters. + :note: + For a description of the applicable parameters, see the documentation of + :meth:`Submodule.add `. - :return: The created submodules. + :return: + The created submodule. """ return Submodule.add(self, *args, **kwargs) @@ -477,7 +505,8 @@ def submodule_update(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: """Update the submodules, keeping the repository consistent as it will take the previous state into consideration. - :note: For more information, please see the documentation of + :note: + For more information, please see the documentation of :meth:`RootModule.update `. """ return RootModule(self).update(*args, **kwargs) @@ -486,16 +515,22 @@ def submodule_update(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: @property def tags(self) -> "IterableList[TagReference]": - """A list of ``Tag`` objects that are available in this repo. + """A list of :class:`~git.refs.tag.TagReference` objects that are available in + this repo. - :return: ``git.IterableList(TagReference, ...)`` + :return: + ``git.IterableList(TagReference, ...)`` """ return TagReference.list_items(self) def tag(self, path: PathLike) -> TagReference: - """:return: TagReference Object, reference pointing to a Commit or Tag + """ + :return: + :class:`~git.refs.tag.TagReference` object, reference pointing to a + :class:`~git.objects.commit.Commit` or tag - :param path: path to the tag reference, i.e. 0.1.5 or tags/0.1.5 + :param path: + Path to the tag reference, e.g. ``0.1.5`` or ``tags/0.1.5``. """ full_path = self._to_full_tag_path(path) return TagReference(self, full_path) @@ -522,14 +557,16 @@ def create_head( :note: For more documentation, please see the :meth:`Head.create ` method. - :return: Newly created :class:`~git.refs.head.Head` Reference + :return: + Newly created :class:`~git.refs.head.Head` Reference. """ return Head.create(self, path, commit, logmsg, force) def delete_head(self, *heads: "Union[str, Head]", **kwargs: Any) -> None: """Delete the given heads. - :param kwargs: Additional keyword arguments to be passed to git-branch + :param kwargs: + Additional keyword arguments to be passed to ``git branch``. """ return Head.delete(self, *heads, **kwargs) @@ -543,10 +580,12 @@ def create_tag( ) -> TagReference: """Create a new tag reference. - :note: For more documentation, please see the + :note: + For more documentation, please see the :meth:`TagReference.create ` method. - :return: :class:`~git.refs.tag.TagReference` object + :return: + :class:`~git.refs.tag.TagReference` object """ return TagReference.create(self, path, ref, message, force, **kwargs) @@ -560,7 +599,8 @@ def create_remote(self, name: str, url: str, **kwargs: Any) -> Remote: For more information, please see the documentation of the :meth:`Remote.create ` method. - :return: :class:`~git.remote.Remote` reference + :return: + :class:`~git.remote.Remote` reference """ return Remote.create(self, name, url, **kwargs) @@ -612,7 +652,8 @@ def config_reader( applicable levels will be used. Specify a level in case you know which file you wish to read to prevent reading multiple files. - :note: On Windows, system configuration cannot currently be read as the path is + :note: + On Windows, system configuration cannot currently be read as the path is unknown, instead the global path will be used. """ return self._config_reader(config_level=config_level) @@ -650,37 +691,43 @@ def config_writer(self, config_level: Lit_config_levels = "repository") -> GitCo return GitConfigParser(self._get_config_path(config_level), read_only=False, repo=self, merge_includes=False) def commit(self, rev: Union[str, Commit_ish, None] = None) -> Commit: - """The Commit object for the specified revision. + """The :class:`~git.objects.commit.Commit` object for the specified revision. + + :param rev: + Revision specifier, see ``git rev-parse`` for viable options. - :param rev: revision specifier, see git-rev-parse for viable options. - :return: :class:`git.Commit ` + :return: + :class:`~git.objects.commit.Commit` """ if rev is None: return self.head.commit return self.rev_parse(str(rev) + "^0") def iter_trees(self, *args: Any, **kwargs: Any) -> Iterator["Tree"]: - """:return: Iterator yielding Tree objects + """:return: Iterator yielding :class:`~git.objects.tree.Tree` objects - :note: Accepts all arguments known to the :meth:`iter_commits` method. + :note: + Accepts all arguments known to the :meth:`iter_commits` method. """ return (c.tree for c in self.iter_commits(*args, **kwargs)) def tree(self, rev: Union[Tree_ish, str, None] = None) -> "Tree": - """The Tree object for the given tree-ish revision. + """The :class:`~git.objects.tree.Tree` object for the given tree-ish revision. Examples:: repo.tree(repo.heads[0]) - :param rev: is a revision pointing to a Treeish (being a commit or tree) + :param rev: + A revision pointing to a Treeish (being a commit or tree). - :return: ``git.Tree`` + :return: + :class:`~git.objects.tree.Tree` :note: - If you need a non-root level tree, find it by iterating the root tree. Otherwise - it cannot know about its path relative to the repository root and subsequent - operations might have unexpected results. + If you need a non-root level tree, find it by iterating the root tree. + Otherwise it cannot know about its path relative to the repository root and + subsequent operations might have unexpected results. """ if rev is None: return self.head.commit.tree @@ -692,23 +739,27 @@ def iter_commits( paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any, ) -> Iterator[Commit]: - """An iterator of Commit objects representing the history of a given ref/commit. + """An iterator of :class:`~git.objects.commit.Commit` objects representing the + history of a given ref/commit. :param rev: - Revision specifier, see git-rev-parse for viable options. + Revision specifier, see ``git rev-parse`` for viable options. If None, the active branch will be used. :param paths: - An optional path or a list of paths; if set only commits that include the - path or paths will be returned + An optional path or a list of paths. If set, only commits that include the + path or paths will be returned. :param kwargs: - Arguments to be passed to git-rev-list - common ones are max_count and skip. + Arguments to be passed to ``git rev-list``. + Common ones are ``max_count`` and ``skip``. - :note: To receive only commits between two named revisions, use the + :note: + To receive only commits between two named revisions, use the ``"revA...revB"`` revision specifier. - :return: Iterator of ``git.Commit`` + :return: + Iterator of :class:`~git.objects.commit.Commit` objects """ if rev is None: rev = self.head.commit @@ -716,16 +767,25 @@ def iter_commits( return Commit.iter_items(self, rev, paths, **kwargs) def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: - """Find the closest common ancestor for the given revision (Commits, Tags, References, etc.). + R"""Find the closest common ancestor for the given revision + (:class:`~git.objects.commit.Commit`\s, :class:`~git.refs.tag.Tag`\s, + :class:`git.refs.reference.Reference`\s, etc.). + + :param rev: + At least two revs to find the common ancestor for. + + :param kwargs: + Additional arguments to be passed to the ``repo.git.merge_base()`` command + which does all the work. - :param rev: At least two revs to find the common ancestor for. - :param kwargs: Additional arguments to be passed to the - ``repo.git.merge_base()`` command which does all the work. - :return: A list of :class:`~git.objects.commit.Commit` objects. If ``--all`` was + :return: + A list of :class:`~git.objects.commit.Commit` objects. If ``--all`` was not passed as a keyword argument, the list will have at max one :class:`~git.objects.commit.Commit`, or is empty if no common merge base exists. - :raises ValueError: If not at least two revs are provided. + + :raises ValueError: + If fewer than two revisions are provided. """ if len(rev) < 2: raise ValueError("Please specify at least two revs, got only %i" % len(rev)) @@ -752,9 +812,14 @@ def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: def is_ancestor(self, ancestor_rev: "Commit", rev: "Commit") -> bool: """Check if a commit is an ancestor of another. - :param ancestor_rev: Rev which should be an ancestor - :param rev: Rev to test against ancestor_rev - :return: ``True``, ancestor_rev is an ancestor to rev. + :param ancestor_rev: + Rev which should be an ancestor. + + :param rev: + Rev to test against `ancestor_rev`. + + :return: + True if `ancestor_rev` is an ancestor to `rev`. """ try: self.git.merge_base(ancestor_rev, rev, is_ancestor=True) @@ -809,7 +874,8 @@ def _set_daemon_export(self, value: object) -> None: def _get_alternates(self) -> List[str]: """The list of alternates for this repo from which objects can be retrieved. - :return: List of strings being pathnames of alternates + :return: + List of strings being pathnames of alternates """ if self.git_dir: alternates_path = osp.join(self.git_dir, "objects", "info", "alternates") @@ -821,17 +887,17 @@ def _get_alternates(self) -> List[str]: return [] def _set_alternates(self, alts: List[str]) -> None: - """Sets the alternates. + """Set the alternates. :param alts: - is the array of string paths representing the alternates at which - git should look for objects, i.e. /home/user/repo/.git/objects + The array of string paths representing the alternates at which git should + look for objects, i.e. ``/home/user/repo/.git/objects``. :raise NoSuchPathError: :note: - The method does not check for the existence of the paths in alts - as the caller is responsible. + The method does not check for the existence of the paths in `alts`, as the + caller is responsible. """ alternates_path = osp.join(self.common_dir, "objects", "info", "alternates") if not alts: @@ -857,9 +923,9 @@ def is_dirty( ) -> bool: """ :return: - ``True`` if the repository is considered dirty. By default it will react - like a git-status without untracked files, hence it is dirty if the - index or the working copy have changes. + True if the repository is considered dirty. By default it will react like a + git-status without untracked files, hence it is dirty if the index or the + working copy have changes. """ if self._bare: # Bare repositories with no associated working directory are @@ -894,11 +960,12 @@ def untracked_files(self) -> List[str]: :return: list(str,...) - Files currently untracked as they have not been staged yet. Paths - are relative to the current working directory of the git command. + Files currently untracked as they have not been staged yet. Paths are + relative to the current working directory of the git command. :note: Ignored files will not appear here, i.e. files mentioned in ``.gitignore``. + :note: This property is expensive, as no cache is involved. To process the result, please consider caching it yourself. @@ -926,23 +993,25 @@ def _get_untracked_files(self, *args: Any, **kwargs: Any) -> List[str]: return untracked_files def ignored(self, *paths: PathLike) -> List[str]: - """Checks if paths are ignored via .gitignore. + """Checks if paths are ignored via ``.gitignore``. This does so using the ``git check-ignore`` method. - :param paths: List of paths to check whether they are ignored or not + :param paths: + List of paths to check whether they are ignored or not. - :return: Subset of those paths which are ignored + :return: + Subset of those paths which are ignored """ try: proc: str = self.git.check_ignore(*paths) except GitCommandError as err: - # If return code is 1, this means none of the items in *paths - # are ignored by Git, so return an empty list. Raise the - # exception on all other return codes. if err.status == 1: + # If return code is 1, this means none of the items in *paths are + # ignored by Git, so return an empty list. return [] else: + # Raise the exception on all other return codes. raise return proc.replace("\\\\", "\\").replace('"', "").split("\n") @@ -951,8 +1020,11 @@ def ignored(self, *paths: PathLike) -> List[str]: def active_branch(self) -> Head: """The name of the currently active branch. - :raises TypeError: If HEAD is detached - :return: Head to the active branch + :raises TypeError: + If HEAD is detached. + + :return: + Head to the active branch """ # reveal_type(self.head.reference) # => Reference return self.head.reference @@ -963,13 +1035,15 @@ def blame_incremental(self, rev: str | HEAD | None, file: str, **kwargs: Any) -> Unlike :meth:`blame`, this does not return the actual file's contents, only a stream of :class:`BlameEntry` tuples. - :param rev: Revision specifier. If `None`, the blame will include all the latest - uncommitted changes. Otherwise, anything succesfully parsed by git-rev-parse - is a valid option. + :param rev: + Revision specifier. If None, the blame will include all the latest + uncommitted changes. Otherwise, anything successfully parsed by ``git + rev-parse`` is a valid option. - :return: Lazy iterator of :class:`BlameEntry` tuples, where the commit indicates - the commit to blame for the line, and range indicates a span of line numbers - in the resulting file. + :return: + Lazy iterator of :class:`BlameEntry` tuples, where the commit indicates the + commit to blame for the line, and range indicates a span of line numbers in + the resulting file. If you combine all line number ranges outputted by this command, you should get a continuous range spanning all line numbers in the file. @@ -981,7 +1055,8 @@ def blame_incremental(self, rev: str | HEAD | None, file: str, **kwargs: Any) -> stream = (line for line in data.split(b"\n") if line) while True: try: - line = next(stream) # When exhausted, causes a StopIteration, terminating this function. + # When exhausted, causes a StopIteration, terminating this function. + line = next(stream) except StopIteration: return split_line = line.split() @@ -990,8 +1065,8 @@ def blame_incremental(self, rev: str | HEAD | None, file: str, **kwargs: Any) -> num_lines = int(num_lines_b) orig_lineno = int(orig_lineno_b) if hexsha not in commits: - # Now read the next few lines and build up a dict of properties - # for this commit. + # Now read the next few lines and build up a dict of properties for this + # commit. props: Dict[bytes, bytes] = {} while True: try: @@ -999,8 +1074,8 @@ def blame_incremental(self, rev: str | HEAD | None, file: str, **kwargs: Any) -> except StopIteration: return if line == b"boundary": - # "boundary" indicates a root commit and occurs - # instead of the "previous" tag. + # "boundary" indicates a root commit and occurs instead of the + # "previous" tag. continue tag, value = line.split(b" ", 1) @@ -1026,11 +1101,12 @@ def blame_incremental(self, rev: str | HEAD | None, file: str, **kwargs: Any) -> ) commits[hexsha] = c else: - # Discard all lines until we find "filename" which is - # guaranteed to be the last line. + # Discard all lines until we find "filename" which is guaranteed to be + # the last line. while True: try: - line = next(stream) # Will fail if we reach the EOF unexpectedly. + # Will fail if we reach the EOF unexpectedly. + line = next(stream) except StopIteration: return tag, value = line.split(b" ", 1) @@ -1055,16 +1131,18 @@ def blame( ) -> List[List[Commit | List[str | bytes] | None]] | Iterator[BlameEntry] | None: """The blame information for the given file at the given revision. - :param rev: Revision specifier. If `None`, the blame will include all the latest - uncommitted changes. Otherwise, anything succesfully parsed by git-rev-parse - is a valid option. + :param rev: + Revision specifier. If None, the blame will include all the latest + uncommitted changes. Otherwise, anything successfully parsed by + git-rev-parse is a valid option. :return: list: [git.Commit, list: []] - A list of lists associating a Commit object with a list of lines that - changed within the given commit. The Commit objects will be given in order - of appearance. + A list of lists associating a :class:`~git.objects.commit.Commit` object + with a list of lines that changed within the given commit. The + :class:`~git.objects.commit.Commit` objects will be given in order of + appearance. """ if incremental: return self.blame_incremental(rev, file, **kwargs) @@ -1096,10 +1174,11 @@ class InfoTD(TypedDict, total=False): parts = [] is_binary = True else: - # As we don't have an idea when the binary data ends, as it could contain multiple newlines - # in the process. So we rely on being able to decode to tell us what it is. - # This can absolutely fail even on text files, but even if it does, we should be fine treating it - # as binary instead. + # As we don't have an idea when the binary data ends, as it could + # contain multiple newlines in the process. So we rely on being able to + # decode to tell us what it is. This can absolutely fail even on text + # files, but even if it does, we should be fine treating it as binary + # instead. parts = self.re_whitespace.split(line_str, 1) firstpart = parts[0] is_binary = False @@ -1180,10 +1259,12 @@ class InfoTD(TypedDict, total=False): line = line_str else: line = line_bytes - # NOTE: We are actually parsing lines out of binary data, which can lead to the - # binary being split up along the newline separator. We will append this to the - # blame we are currently looking at, even though it should be concatenated with - # the last line we have seen. + # NOTE: We are actually parsing lines out of binary + # data, which can lead to the binary being split up + # along the newline separator. We will append this + # to the blame we are currently looking at, even + # though it should be concatenated with the last + # line we have seen. blames[-1][1].append(line) info = {"id": sha} @@ -1205,28 +1286,30 @@ def init( """Initialize a git repository at the given path if specified. :param path: - The full path to the repo (traditionally ends with /.git) or None in - which case the repository will be created in the current working directory + The full path to the repo (traditionally ends with ``/.git``). + Or None, in which case the repository will be created in the current working + directory. :param mkdir: - If specified, will create the repository directory if it doesn't - already exist. Creates the directory with a mode=0755. + If specified, will create the repository directory if it doesn't already + exist. Creates the directory with a mode=0755. Only effective if a path is explicitly given. :param odbt: - Object DataBase type - a type which is constructed by providing - the directory containing the database objects, i.e. .git/objects. - It will be used to access all object data. + Object DataBase type - a type which is constructed by providing the + directory containing the database objects, i.e. ``.git/objects``. It will be + used to access all object data. :param expand_vars: - If specified, environment variables will not be escaped. This - can lead to information disclosure, allowing attackers to - access the contents of environment variables. + If specified, environment variables will not be escaped. This can lead to + information disclosure, allowing attackers to access the contents of + environment variables. :param kwargs: - Keyword arguments serving as additional options to the git-init command. + Keyword arguments serving as additional options to the ``git init`` command. - :return: ``git.Repo`` (the newly created repo) + :return: + :class:`Repo` (the newly created repo) """ if path: path = expand_path(path, expand_vars) @@ -1253,7 +1336,7 @@ def _clone( ) -> "Repo": odbt = kwargs.pop("odbt", odb_default_type) - # When pathlib.Path or other classbased path is passed + # When pathlib.Path or other class-based path is passed if not isinstance(path, str): path = str(path) @@ -1315,11 +1398,10 @@ def _clone( # Retain env values that were passed to _clone(). repo.git.update_environment(**git.environment()) - # Adjust remotes - there may be operating systems which use backslashes, - # These might be given as initial paths, but when handling the config file - # that contains the remote from which we were clones, git stops liking it - # as it will escape the backslashes. Hence we undo the escaping just to be - # sure. + # Adjust remotes - there may be operating systems which use backslashes, These + # might be given as initial paths, but when handling the config file that + # contains the remote from which we were clones, git stops liking it as it will + # escape the backslashes. Hence we undo the escaping just to be sure. if repo.remotes: with repo.remotes[0].config_writer as writer: writer.set_value("url", Git.polish_url(repo.remotes[0].url)) @@ -1337,21 +1419,38 @@ def clone( ) -> "Repo": """Create a clone from this repository. - :param path: The full path of the new repo (traditionally ends with - ``./.git``). - :param progress: See :meth:`git.remote.Remote.push`. - :param multi_options: A list of Clone options that can be provided multiple times. + :param path: + The full path of the new repo (traditionally ends with ``./.git``). + + :param progress: + See :meth:`Remote.push `. + + :param multi_options: + A list of ``git clone`` options that can be provided multiple times. + One option per list item which is passed exactly as specified to clone. - For example: ['--config core.filemode=false', '--config core.ignorecase', - '--recurse-submodule=repo1_path', '--recurse-submodule=repo2_path'] - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack. + For example:: + + [ + "--config core.filemode=false", + "--config core.ignorecase", + "--recurse-submodule=repo1_path", + "--recurse-submodule=repo2_path", + ] + + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :param allow_unsafe_options: + Allow unsafe options to be used, like ``--upload-pack``. + :param kwargs: * odbt = ObjectDatabase Type, allowing to determine the object database implementation used by the returned Repo instance. - * All remaining keyword arguments are given to the git-clone command. + * All remaining keyword arguments are given to the ``git clone`` command. - :return: :class:`Repo` (the newly cloned repo) + :return: + :class:`Repo` (the newly cloned repo) """ return self._clone( self.git, @@ -1379,29 +1478,38 @@ def clone_from( ) -> "Repo": """Create a clone from the given URL. - :param url: Valid git url, see http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS + :param url: + Valid git url, see: + http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS - :param to_path: Path to which the repository should be cloned to. + :param to_path: + Path to which the repository should be cloned to. - :param progress: See :meth:`git.remote.Remote.push`. + :param progress: + See :meth:`Remote.push `. - :param env: Optional dictionary containing the desired environment variables. + :param env: + Optional dictionary containing the desired environment variables. - Note: Provided variables will be used to update the execution - environment for `git`. If some variable is not specified in `env` - and is defined in `os.environ`, value from `os.environ` will be used. - If you want to unset some variable, consider providing empty string - as its value. + Note: Provided variables will be used to update the execution environment + for ``git``. If some variable is not specified in `env` and is defined in + :attr:`os.environ`, value from :attr:`os.environ` will be used. If you want + to unset some variable, consider providing empty string as its value. - :param multi_options: See :meth:`clone` method. + :param multi_options: + See the :meth:`clone` method. - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack. + :param allow_unsafe_options: + Allow unsafe options to be used, like ``--upload-pack``. - :param kwargs: See the :meth:`clone` method. + :param kwargs: + See the :meth:`clone` method. - :return: :class:`Repo` instance pointing to the cloned directory. + :return: + :class:`Repo` instance pointing to the cloned directory. """ git = cls.GitCommandWrapperType(os.getcwd()) if env is not None: @@ -1459,10 +1567,14 @@ def archive( def has_separate_working_tree(self) -> bool: """ - :return: True if our git_dir is not at the root of our working_tree_dir, but a .git file with a - platform agnositic symbolic link. Our git_dir will be wherever the .git file points to. + :return: + True if our :attr:`git_dir` is not at the root of our + :attr:`working_tree_dir`, but a ``.git`` file with a platform-agnostic + symbolic link. Our :attr:`git_dir` will be wherever the ``.git`` file points + to. - :note: bare repositories will always return False here + :note: + Bare repositories will always return False here. """ if self.bare: return False @@ -1479,7 +1591,8 @@ def __repr__(self) -> str: def currently_rebasing_on(self) -> Commit | None: """ - :return: The commit which is currently being replayed while rebasing. + :return: + The commit which is currently being replayed while rebasing. None if we are not currently rebasing. """ diff --git a/git/repo/fun.py b/git/repo/fun.py index 63bcfdfc7..1a53a6825 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -57,13 +57,13 @@ def touch(filename: str) -> str: def is_git_dir(d: "PathLike") -> bool: - """This is taken from the git setup.c:is_git_directory - function. + """This is taken from the git setup.c:is_git_directory function. - @throws WorkTreeRepositoryUnsupported if it sees a worktree directory. It's quite hacky to do that here, - but at least clearly indicates that we don't support it. - There is the unlikely danger to throw if we see directories which just look like a worktree dir, - but are none.""" + :raises WorkTreeRepositoryUnsupported: + If it sees a worktree directory. It's quite hacky to do that here, but at least + clearly indicates that we don't support it. There is the unlikely danger to + throw if we see directories which just look like a worktree dir, but are none. + """ if osp.isdir(d): if (osp.isdir(osp.join(d, "objects")) or "GIT_OBJECT_DIRECTORY" in os.environ) and osp.isdir( osp.join(d, "refs") @@ -107,15 +107,15 @@ def find_submodule_git_dir(d: "PathLike") -> Optional["PathLike"]: with open(d) as fp: content = fp.read().rstrip() except IOError: - # it's probably not a file + # It's probably not a file. pass else: if content.startswith("gitdir: "): path = content[8:] if Git.is_cygwin(): - ## Cygwin creates submodules prefixed with `/cygdrive/...` suffixes. - # Cygwin git understands Cygwin paths much better than Windows ones + # Cygwin creates submodules prefixed with `/cygdrive/...` suffixes. + # Cygwin git understands Cygwin paths much better than Windows ones. # Also the Cygwin tests are assuming Cygwin paths. path = cygpath(path) if not osp.isabs(path): @@ -126,9 +126,14 @@ def find_submodule_git_dir(d: "PathLike") -> Optional["PathLike"]: def short_to_long(odb: "GitCmdObjectDB", hexsha: str) -> Optional[bytes]: - """:return: long hexadecimal sha1 from the given less-than-40 byte hexsha - or None if no candidate could be found. - :param hexsha: hexsha with less than 40 byte""" + """ + :return: + Long hexadecimal sha1 from the given less than 40 byte hexsha or None if no + candidate could be found. + + :param hexsha: + hexsha with less than 40 bytes. + """ try: return bin_to_hex(odb.partial_to_complete_sha_hex(hexsha)) except BadObject: @@ -140,25 +145,29 @@ def name_to_object( repo: "Repo", name: str, return_ref: bool = False ) -> Union[SymbolicReference, "Commit", "TagObject", "Blob", "Tree"]: """ - :return: object specified by the given name, hexshas ( short and long ) - as well as references are supported - :param return_ref: if name specifies a reference, we will return the reference - instead of the object. Otherwise it will raise BadObject or BadName + :return: + Object specified by the given name - hexshas (short and long) as well as + references are supported. + + :param return_ref: + If True, and name specifies a reference, we will return the reference + instead of the object. Otherwise it will raise `~gitdb.exc.BadObject` o + `~gitdb.exc.BadName`. """ hexsha: Union[None, str, bytes] = None - # is it a hexsha ? Try the most common ones, which is 7 to 40 + # Is it a hexsha? Try the most common ones, which is 7 to 40. if repo.re_hexsha_shortened.match(name): if len(name) != 40: - # find long sha for short sha + # Find long sha for short sha. hexsha = short_to_long(repo.odb, name) else: hexsha = name # END handle short shas # END find sha if it matches - # if we couldn't find an object for what seemed to be a short hexsha - # try to find it as reference anyway, it could be named 'aaa' for instance + # If we couldn't find an object for what seemed to be a short hexsha, try to find it + # as reference anyway, it could be named 'aaa' for instance. if hexsha is None: for base in ( "%s", @@ -179,12 +188,12 @@ def name_to_object( # END for each base # END handle hexsha - # didn't find any ref, this is an error + # Didn't find any ref, this is an error. if return_ref: raise BadObject("Couldn't find reference named %r" % name) # END handle return ref - # tried everything ? fail + # Tried everything ? fail. if hexsha is None: raise BadName(name) # END assert hexsha was found @@ -216,17 +225,27 @@ def to_commit(obj: Object) -> Union["Commit", "TagObject"]: def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: """ - :return: Object at the given revision, either Commit, Tag, Tree or Blob - :param rev: git-rev-parse compatible revision specification as string, please see - http://www.kernel.org/pub/software/scm/git/docs/git-rev-parse.html - for details - :raise BadObject: if the given revision could not be found - :raise ValueError: If rev couldn't be parsed - :raise IndexError: If invalid reflog index is specified""" - - # colon search mode ? + :return: + `~git.objects.base.Object` at the given revision, either + `~git.objects.commit.Commit`, `~git.refs.tag.Tag`, `~git.objects.tree.Tree` or + `~git.objects.blob.Blob`. + + :param rev: + ``git rev-parse``-compatible revision specification as string. Please see + http://www.kernel.org/pub/software/scm/git/docs/git-rev-parse.html for details. + + :raise BadObject: + If the given revision could not be found. + + :raise ValueError: + If `rev` couldn't be parsed. + + :raise IndexError: + If an invalid reflog index is specified. + """ + # Are we in colon search mode? if rev.startswith(":/"): - # colon search mode + # Colon search mode raise NotImplementedError("commit by message search ( regex )") # END handle search @@ -245,7 +264,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: token = rev[start] if obj is None: - # token is a rev name + # token is a rev name. if start == 0: ref = repo.head.ref else: @@ -265,29 +284,29 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: start += 1 - # try to parse {type} + # Try to parse {type}. if start < lr and rev[start] == "{": end = rev.find("}", start) if end == -1: raise ValueError("Missing closing brace to define type in %s" % rev) - output_type = rev[start + 1 : end] # exclude brace + output_type = rev[start + 1 : end] # Exclude brace. - # handle type + # Handle type. if output_type == "commit": - pass # default + pass # Default. elif output_type == "tree": try: obj = cast(Commit_ish, obj) obj = to_commit(obj).tree except (AttributeError, ValueError): - pass # error raised later + pass # Error raised later. # END exception handling elif output_type in ("", "blob"): obj = cast("TagObject", obj) if obj and obj.type == "tag": obj = deref_tag(obj) else: - # cannot do anything for non-tags + # Cannot do anything for non-tags. pass # END handle tag elif token == "@": @@ -295,11 +314,10 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: assert ref is not None, "Require Reference to access reflog" revlog_index = None try: - # transform reversed index into the format of our revlog + # Transform reversed index into the format of our revlog. revlog_index = -(int(output_type) + 1) except ValueError as e: - # TODO: Try to parse the other date options, using parse_date - # maybe + # TODO: Try to parse the other date options, using parse_date maybe. raise NotImplementedError("Support for additional @{...} modes not implemented") from e # END handle revlog index @@ -311,23 +329,24 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: obj = Object.new_from_sha(repo, hex_to_bin(entry.newhexsha)) - # make it pass the following checks + # Make it pass the following checks. output_type = "" else: raise ValueError("Invalid output type: %s ( in %s )" % (output_type, rev)) # END handle output type - # empty output types don't require any specific type, its just about dereferencing tags + # Empty output types don't require any specific type, its just about + # dereferencing tags. if output_type and obj and obj.type != output_type: raise ValueError("Could not accommodate requested object type %r, got %s" % (output_type, obj.type)) # END verify output type - start = end + 1 # skip brace + start = end + 1 # Skip brace. parsed_to = start continue # END parse type - # try to parse a number + # Try to parse a number. num = 0 if token != ":": found_digit = False @@ -341,15 +360,14 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: # END handle number # END number parse loop - # no explicit number given, 1 is the default - # It could be 0 though + # No explicit number given, 1 is the default. It could be 0 though. if not found_digit: num = 1 # END set default num # END number parsing only if non-blob mode parsed_to = start - # handle hierarchy walk + # Handle hierarchy walk. try: obj = cast(Commit_ish, obj) if token == "~": @@ -359,7 +377,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: # END for each history item to walk elif token == "^": obj = to_commit(obj) - # must be n'th parent + # Must be n'th parent. if num: obj = obj.parents[num - 1] elif token == ":": @@ -378,7 +396,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: # END exception handling # END parse loop - # still no obj ? Its probably a simple name + # Still no obj? It's probably a simple name. if obj is None: obj = cast(Commit_ish, name_to_object(repo, rev)) parsed_to = lr From c67b2e2a05be13c70a72147b9553348cdc5217a9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 19:55:36 -0500 Subject: [PATCH 0690/1392] Adjust spacing in colon seach mode NotImplementedError This is a (very) minor improvement that uses the more common convention of not padding parentheses on the inside with spaces in prose, in an exception message. --- git/repo/fun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/repo/fun.py b/git/repo/fun.py index 1a53a6825..b080cac5b 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -246,7 +246,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: # Are we in colon search mode? if rev.startswith(":/"): # Colon search mode - raise NotImplementedError("commit by message search ( regex )") + raise NotImplementedError("commit by message search (regex)") # END handle search obj: Union[Commit_ish, "Reference", None] = None From 5ee87441a5c936a55c69e310e4c8812b635cbdf0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 20:10:33 -0500 Subject: [PATCH 0691/1392] Update git source link in Repo.merge_base comment And link to a specific tag (the most recent stable version tag) so the the hyperlink won't break in the future (as long as GitHub URLs keep working). --- git/repo/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index afbc1d75d..a0266f090 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -798,8 +798,8 @@ def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: if err.status == 128: raise # END handle invalid rev - # Status code 1 is returned if there is no merge-base - # (see https://github.com/git/git/blob/master/builtin/merge-base.c#L16) + # Status code 1 is returned if there is no merge-base. + # (See: https://github.com/git/git/blob/v2.44.0/builtin/merge-base.c#L19) return res # END exception handling From c8b6cf0cd96e6c0129aea563cb5092d6658f4424 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 20:32:14 -0500 Subject: [PATCH 0692/1392] Update comment about improving expand_path overloads --- git/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index 72861ef84..4929affeb 100644 --- a/git/util.py +++ b/git/util.py @@ -500,7 +500,7 @@ def expand_path(p: None, expand_vars: bool = ...) -> None: @overload def expand_path(p: PathLike, expand_vars: bool = ...) -> str: - # improve these overloads when 3.5 dropped + # TODO: Support for Python 3.5 has been dropped, so these overloads can be improved. ... From bcc0c27482a46ff030cbddf75feac070f169263e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 20:38:15 -0500 Subject: [PATCH 0693/1392] Fix recent inconsistency, using :raise:, not :raises: GitPython used :raise: consistently before, but I recently introduced some occurrences of :raises: by accident. This changes those to :raise:. --- git/index/fun.py | 2 +- git/repo/base.py | 2 +- git/repo/fun.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/git/index/fun.py b/git/index/fun.py index 785a1c748..a60a3b809 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -88,7 +88,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: :param args: Arguments passed to hook file. - :raises HookExecutionError: + :raise HookExecutionError: """ hp = hook_path(name, index.repo.git_dir) if not os.access(hp, os.X_OK): diff --git a/git/repo/base.py b/git/repo/base.py index a0266f090..1e58a24d0 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -784,7 +784,7 @@ def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: :class:`~git.objects.commit.Commit`, or is empty if no common merge base exists. - :raises ValueError: + :raise ValueError: If fewer than two revisions are provided. """ if len(rev) < 2: diff --git a/git/repo/fun.py b/git/repo/fun.py index b080cac5b..fa2cebb67 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -59,7 +59,7 @@ def touch(filename: str) -> str: def is_git_dir(d: "PathLike") -> bool: """This is taken from the git setup.c:is_git_directory function. - :raises WorkTreeRepositoryUnsupported: + :raise WorkTreeRepositoryUnsupported: If it sees a worktree directory. It's quite hacky to do that here, but at least clearly indicates that we don't support it. There is the unlikely danger to throw if we see directories which just look like a worktree dir, but are none. From 0231b745b0005cb795a1a08493de018590daadef Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 21:09:57 -0500 Subject: [PATCH 0694/1392] Further revise docstrings in git.objects.submodule.base I had missed a lot of stuff there in 63c62ed. --- git/objects/submodule/base.py | 447 ++++++++++++++++++++++------------ 1 file changed, 289 insertions(+), 158 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 58b474fdd..70a021c54 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -133,8 +133,9 @@ def __init__( :param url: The URL to the remote repository which is the submodule. - :param branch_path: Complete relative path to ref to checkout when cloning the - remote repository. + :param branch_path: + Complete relative path to ref to checkout when cloning the remote + repository. """ super().__init__(repo, binsha, mode, path) self.size = 0 @@ -214,10 +215,12 @@ def _config_parser( cls, repo: "Repo", parent_commit: Union[Commit_ish, None], read_only: bool ) -> SubmoduleConfigParser: """ - :return: Config Parser constrained to our submodule in read or write mode + :return: + Config parser constrained to our submodule in read or write mode - :raise IOError: If the .gitmodules file cannot be found, either locally or in - the repository at the given parent commit. Otherwise the exception would be + :raise IOError: + If the ``.gitmodules`` file cannot be found, either locally or in the + repository at the given parent commit. Otherwise the exception would be delayed until the first access of the config parser. """ parent_matches_head = True @@ -225,7 +228,8 @@ def _config_parser( try: parent_matches_head = repo.head.commit == parent_commit except ValueError: - # We are most likely in an empty repository, so the HEAD doesn't point to a valid ref. + # We are most likely in an empty repository, so the HEAD doesn't point + # to a valid ref. pass # END handle parent_commit fp_module: Union[str, BytesIO] @@ -260,13 +264,17 @@ def _clear_cache(self) -> None: @classmethod def _sio_modules(cls, parent_commit: Commit_ish) -> BytesIO: - """:return: Configuration file as BytesIO - we only access it through the respective blob's data""" + """ + :return: + Configuration file as BytesIO - we only access it through the respective + blob's data + """ sio = BytesIO(parent_commit.tree[cls.k_modules_file].data_stream.read()) sio.name = cls.k_modules_file return sio def _config_parser_constrained(self, read_only: bool) -> SectionConstraint: - """:return: Config Parser constrained to our submodule in read or write mode""" + """:return: Config parser constrained to our submodule in read or write mode""" try: pc: Union["Commit_ish", None] = self.parent_commit except ValueError: @@ -296,14 +304,29 @@ def _clone_repo( **kwargs: Any, ) -> "Repo": """ - :return: Repo instance of newly cloned repository - :param repo: Our parent repository - :param url: URL to clone from - :param path: Repository - relative path to the submodule checkout location - :param name: Canonical name of the submodule - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack - :param kwargs: Additional arguments given to git.clone + :return: + :class:`~git.repo.base.Repo` instance of newly cloned repository. + + :param repo: + Our parent repository. + + :param url: + URL to clone from. + + :param path: + Repository-relative path to the submodule checkout location. + + :param name: + Canonical name of the submodule. + + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :param allow_unsafe_options: + Allow unsafe options to be used, like ``--upload-pack``. + + :param kwargs: + Additional arguments given to ``git clone`` """ module_abspath = cls._module_abspath(repo, path, name) module_checkout_path = module_abspath @@ -328,8 +351,11 @@ def _clone_repo( @classmethod def _to_relative_path(cls, parent_repo: "Repo", path: PathLike) -> PathLike: - """:return: a path guaranteed to be relative to the given parent - repository - :raise ValueError: if path is not contained in the parent repository's working tree""" + """:return: a path guaranteed to be relative to the given parent repository + + :raise ValueError: + If path is not contained in the parent repository's working tree + """ path = to_native_path_linux(path) if path.endswith("/"): path = path[:-1] @@ -352,16 +378,26 @@ def _to_relative_path(cls, parent_repo: "Repo", path: PathLike) -> PathLike: @classmethod def _write_git_file_and_module_config(cls, working_tree_dir: PathLike, module_abspath: PathLike) -> None: - """Write a .git file containing a(preferably) relative path to the actual git module repository. + """Write a .git file containing a (preferably) relative path to the actual git + module repository. + + It is an error if the `module_abspath` cannot be made into a relative path, + relative to the `working_tree_dir`. + + :note: + This will overwrite existing files! + + :note: + As we rewrite both the git file as well as the module configuration, we + might fail on the configuration and will not roll back changes done to the + git file. This should be a non-issue, but may easily be fixed if it becomes + one. - It is an error if the module_abspath cannot be made into a relative path, relative to the working_tree_dir + :param working_tree_dir: + Directory to write the ``.git`` file into. - :note: This will overwrite existing files! - :note: as we rewrite both the git file as well as the module configuration, we might fail on the configuration - and will not roll back changes done to the git file. This should be a non - issue, but may easily be fixed - if it becomes one. - :param working_tree_dir: Directory to write the .git file into - :param module_abspath: Absolute path to the bare repository + :param module_abspath: + Absolute path to the bare repository. """ git_file = osp.join(working_tree_dir, ".git") rela_path = osp.relpath(module_abspath, start=working_tree_dir) @@ -395,54 +431,77 @@ def add( allow_unsafe_protocols: bool = False, ) -> "Submodule": """Add a new submodule to the given repository. This will alter the index - as well as the .gitmodules file, but will not create a new commit. + as well as the ``.gitmodules`` file, but will not create a new commit. If the submodule already exists, no matter if the configuration differs from the one provided, the existing submodule will be returned. - :param repo: Repository instance which should receive the submodule. - :param name: The name/identifier for the submodule. - :param path: Repository-relative or absolute path at which the submodule - should be located. + :param repo: + Repository instance which should receive the submodule. + + :param name: + The name/identifier for the submodule. + + :param path: + Repository-relative or absolute path at which the submodule should be + located. It will be created as required during the repository initialization. - :param url: git-clone compatible URL, see git-clone reference for more information. - If None, the repository is assumed to exist, and the url of the first - remote is taken instead. This is useful if you want to make an existing - repository a submodule of another one. - :param branch: name of branch at which the submodule should (later) be checked out. - The given branch must exist in the remote repository, and will be checked - out locally as a tracking branch. - It will only be written into the configuration if it not None, which is - when the checked out branch will be the one the remote HEAD pointed to. - The result you get in these situation is somewhat fuzzy, and it is recommended - to specify at least 'master' here. - Examples are 'master' or 'feature/new'. - :param no_checkout: If True, and if the repository has to be cloned manually, - no checkout will be performed. - :param depth: Create a shallow clone with a history truncated to the - specified number of commits. - :param env: Optional dictionary containing the desired environment variables. + + :param url: + git-clone compatible URL, see git-clone reference for more information. + If None, the repository is assumed to exist, and the url of the first remote + is taken instead. This is useful if you want to make an existing repository + a submodule of another one. + + :param branch: + Name of branch at which the submodule should (later) be checked out. The + given branch must exist in the remote repository, and will be checked out + locally as a tracking branch. + It will only be written into the configuration if it not None, which is when + the checked out branch will be the one the remote HEAD pointed to. + The result you get in these situation is somewhat fuzzy, and it is + recommended to specify at least ``master`` here. + Examples are ``master`` or ``feature/new``. + + :param no_checkout: + If True, and if the repository has to be cloned manually, no checkout will + be performed. + + :param depth: + Create a shallow clone with a history truncated to the specified number of + commits. + + :param env: + Optional dictionary containing the desired environment variables. + Note: Provided variables will be used to update the execution environment for ``git``. If some variable is not specified in `env` and is defined in attr:`os.environ`, the value from attr:`os.environ` will be used. If you want to unset some variable, consider providing an empty string as its value. + :param clone_multi_options: A list of Clone options. Please see :meth:`Repo.clone ` for details. - :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ext. - :param allow_unsafe_options: Allow unsafe options to be used, like --upload-pack - :return: The newly created submodule instance. - :note: Works atomically, such that no change will be done if the repository - update fails for instance. - """ + :param allow_unsafe_protocols: + Allow unsafe protocols to be used, like ``ext``. + + :param allow_unsafe_options: + Allow unsafe options to be used, like ``--upload-pack``. + + :return: + The newly created :class:`Submodule` instance. + + :note: + Works atomically, such that no change will be done if, for example, the + repository update fails. + """ if repo.bare: raise InvalidGitRepositoryError("Cannot add submodules to bare repositories") # END handle bare repos path = cls._to_relative_path(repo, path) - # Ensure we never put backslashes into the URL, as some operating systems - # like it... + # Ensure we never put backslashes into the URL, as might happen on Windows. if url is not None: url = to_native_path_linux(url) # END ensure URL correctness @@ -569,24 +628,29 @@ def update( allow_unsafe_options: bool = False, allow_unsafe_protocols: bool = False, ) -> "Submodule": - """Update the repository of this submodule to point to the checkout - we point at with the binsha of this instance. + """Update the repository of this submodule to point to the checkout we point at + with the binsha of this instance. :param recursive: If True, we will operate recursively and update child modules as well. + :param init: If True, the module repository will be cloned into place if necessary. + :param to_latest_revision: If True, the submodule's sha will be ignored during checkout. Instead, the remote will be fetched, and the local tracking branch updated. This only works if we have a local tracking branch, which is the case if the remote repository had a master branch, or of the 'branch' option was specified for this submodule and the branch existed remotely. + :param progress: - UpdateProgress instance or None if no progress should be shown. + :class:`UpdateProgress` instance, or None if no progress should be shown. + :param dry_run: If True, the operation will only be simulated, but not performed. All performed operations are read-only. + :param force: If True, we may reset heads even if the repository in question is dirty. Additionally we will be allowed to set a tracking branch which is ahead of @@ -594,31 +658,40 @@ def update( This will essentially 'forget' commits. If False, local tracking branches that are in the future of their respective remote branches will simply not be moved. + :param keep_going: If True, we will ignore but log all errors, and keep going recursively. - Unless dry_run is set as well, keep_going could cause subsequent / inherited - errors you wouldn't see otherwise. - In conjunction with dry_run, it can be useful to anticipate all errors when - updating submodules. + Unless `dry_run` is set as well, `keep_going` could cause + subsequent/inherited errors you wouldn't see otherwise. + In conjunction with `dry_run`, it can be useful to anticipate all errors + when updating submodules. + :param env: Optional dictionary containing the desired environment variables. Note: Provided variables will be used to update the execution environment for ``git``. If some variable is not specified in `env` and is defined in attr:`os.environ`, value from attr:`os.environ` will be used. If you want to unset some variable, consider providing the empty string as its value. + :param clone_multi_options: - List of Clone options. + List of ``git clone`` options. Please see :meth:`Repo.clone ` for details. They only take effect with the `init` option. + :param allow_unsafe_protocols: - Allow unsafe protocols to be used, like ext. + Allow unsafe protocols to be used, like ``ext``. + :param allow_unsafe_options: - Allow unsafe options to be used, like --upload-pack. + Allow unsafe options to be used, like ``--upload-pack``. - :note: Does nothing in bare repositories. - :note: This method is definitely not atomic if `recursive` is True. + :note: + Does nothing in bare repositories. - :return: self + :note: + This method is definitely not atomic if `recursive` is True. + + :return: + self """ if self.repo.bare: return self @@ -742,9 +815,10 @@ def update( # END handle tracking branch # NOTE: Have to write the repo config file as well, otherwise the - # default implementation will be offended and not update the repository. - # Maybe this is a good way to ensure it doesn't get into our way, but - # we want to stay backwards compatible too... It's so redundant! + # default implementation will be offended and not update the + # repository. Maybe this is a good way to ensure it doesn't get into + # our way, but we want to stay backwards compatible too... It's so + # redundant! with self.repo.config_writer() as writer: writer.set_value(sm_section(self.name), "url", self.url) # END handle dry_run @@ -755,7 +829,8 @@ def update( binsha = self.binsha hexsha = self.hexsha if mrepo is not None: - # mrepo is only set if we are not in dry-run mode or if the module existed. + # mrepo is only set if we are not in dry-run mode or if the module + # existed. is_detached = mrepo.head.is_detached # END handle dry_run @@ -782,10 +857,12 @@ def update( # Update the working tree. # Handles dry_run. if mrepo is not None and mrepo.head.commit.binsha != binsha: - # We must ensure that our destination sha (the one to point to) is in the future of our current head. - # Otherwise, we will reset changes that might have been done on the submodule, but were not yet pushed. - # We also handle the case that history has been rewritten, leaving no merge-base. In that case - # we behave conservatively, protecting possible changes the user had done. + # We must ensure that our destination sha (the one to point to) is in + # the future of our current head. Otherwise, we will reset changes that + # might have been done on the submodule, but were not yet pushed. We + # also handle the case that history has been rewritten, leaving no + # merge-base. In that case we behave conservatively, protecting possible + # changes the user had done. may_reset = True if mrepo.head.commit.binsha != self.NULL_BIN_SHA: base_commit = mrepo.merge_base(mrepo.head.commit, hexsha) @@ -822,10 +899,10 @@ def update( if not dry_run and may_reset: if is_detached: - # NOTE: For now we force. The user is not supposed to change detached - # submodules anyway. Maybe at some point this becomes an option, to - # properly handle user modifications - see below for future options - # regarding rebase and merge. + # NOTE: For now we force. The user is not supposed to change + # detached submodules anyway. Maybe at some point this becomes + # an option, to properly handle user modifications - see below + # for future options regarding rebase and merge. mrepo.git.checkout(hexsha, force=force) else: mrepo.head.reset(hexsha, index=True, working_tree=True) @@ -871,19 +948,30 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = the repository at our current path, changing the configuration, as well as adjusting our index entry accordingly. - :param module_path: The path to which to move our module in the parent - repository's working tree, given as repository - relative or absolute path. - Intermediate directories will be created accordingly. If the path already - exists, it must be empty. Trailing (back)slashes are removed automatically. - :param configuration: If True, the configuration will be adjusted to let - the submodule point to the given path. - :param module: If True, the repository managed by this submodule - will be moved as well. If False, we don't move the submodule's checkout, - which may leave the parent repository in an inconsistent state. - :return: self - :raise ValueError: If the module path existed and was not empty, or was a file. - :note: Currently the method is not atomic, and it could leave the repository - in an inconsistent state if a sub-step fails for some reason. + :param module_path: + The path to which to move our module in the parent repository's working + tree, given as repository - relative or absolute path. Intermediate + directories will be created accordingly. If the path already exists, it must + be empty. Trailing (back)slashes are removed automatically. + + :param configuration: + If True, the configuration will be adjusted to let the submodule point to + the given path. + + :param module: + If True, the repository managed by this submodule will be moved as well. If + False, we don't move the submodule's checkout, which may leave the parent + repository in an inconsistent state. + + :return: + self + + :raise ValueError: + If the module path existed and was not empty, or was a file. + + :note: + Currently the method is not atomic, and it could leave the repository in an + inconsistent state if a sub-step fails for some reason. """ if module + configuration < 1: raise ValueError("You must specify to move at least the module or the configuration of the submodule") @@ -940,8 +1028,8 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = # END handle git file rewrite # END move physical module - # Rename the index entry - we have to manipulate the index directly as - # git-mv cannot be used on submodules... yeah. + # Rename the index entry - we have to manipulate the index directly as git-mv + # cannot be used on submodules... yeah. previous_sm_path = self.path try: if configuration: @@ -967,7 +1055,8 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = raise # END handle undo rename - # Auto-rename submodule if its name was 'default', that is, the checkout directory. + # Auto-rename submodule if its name was 'default', that is, the checkout + # directory. if previous_sm_path == self.name: self.rename(module_checkout_path) @@ -982,30 +1071,46 @@ def remove( dry_run: bool = False, ) -> "Submodule": """Remove this submodule from the repository. This will remove our entry - from the .gitmodules file and the entry in the .git/config file. - - :param module: If True, the checked out module we point to will be deleted as - well. If that module is currently on a commit outside any branch in the - remote, or if it is ahead of its tracking branch, or if there are modified - or untracked files in its working tree, then the removal will fail. In case - the removal of the repository fails for these reasons, the submodule status - will not have been altered. + from the ``.gitmodules`` file and the entry in the ``.git/config`` file. + + :param module: + If True, the checked out module we point to will be deleted as well. If that + module is currently on a commit outside any branch in the remote, or if it + is ahead of its tracking branch, or if there are modified or untracked files + in its working tree, then the removal will fail. In case the removal of the + repository fails for these reasons, the submodule status will not have been + altered. If this submodule has child modules of its own, these will be deleted prior to touching the direct submodule. - :param force: Enforces the deletion of the module even though it contains - modifications. This basically enforces a brute-force file system based - deletion. - :param configuration: If True, the submodule is deleted from the configuration, - otherwise it isn't. Although this should be enabled most of the time, this - flag enables you to safely delete the repository of your submodule. - :param dry_run: If True, we will not actually do anything, but throw the errors - we would usually throw. - :return: self - :note: Doesn't work in bare repositories. - :note: Doesn't work atomically, as failure to remove any part of the submodule - will leave an inconsistent state. - :raise InvalidGitRepositoryError: Thrown if the repository cannot be deleted. - :raise OSError: If directories or files could not be removed. + + :param force: + Enforces the deletion of the module even though it contains modifications. + This basically enforces a brute-force file system based deletion. + + :param configuration: + If True, the submodule is deleted from the configuration, otherwise it + isn't. Although this should be enabled most of the time, this flag enables + you to safely delete the repository of your submodule. + + :param dry_run: + If True, we will not actually do anything, but throw the errors we would + usually throw. + + :return: + self + + :note: + Doesn't work in bare repositories. + + :note: + Doesn't work atomically, as failure to remove any part of the submodule will + leave an inconsistent state. + + :raise InvalidGitRepositoryError: + Thrown if the repository cannot be deleted. + + :raise OSError: + If directories or files could not be removed. """ if not (module or configuration): raise ValueError("Need to specify to delete at least the module, or the configuration") @@ -1019,8 +1124,9 @@ def remove( del csm if configuration and not dry_run and nc > 0: - # Ensure we don't leave the parent repository in a dirty state, and commit our changes. - # It's important for recursive, unforced, deletions to work as expected. + # Ensure we don't leave the parent repository in a dirty state, and commit + # our changes. It's important for recursive, unforced, deletions to work as + # expected. self.module().index.commit("Removed at least one of child-modules of '%s'" % self.name) # END handle recursion @@ -1031,8 +1137,9 @@ def remove( git_dir = mod.git_dir if force: # Take the fast lane and just delete everything in our module path. - # TODO: If we run into permission problems, we have a highly inconsistent - # state. Delete the .git folders last, start with the submodules first. + # TODO: If we run into permission problems, we have a highly + # inconsistent state. Delete the .git folders last, start with the + # submodules first. mp = self.abspath method: Union[None, Callable[[PathLike], None]] = None if osp.islink(mp): @@ -1129,18 +1236,24 @@ def remove( def set_parent_commit(self, commit: Union[Commit_ish, None], check: bool = True) -> "Submodule": """Set this instance to use the given commit whose tree is supposed to - contain the .gitmodules blob. + contain the ``.gitmodules`` blob. :param commit: Commit-ish reference pointing at the root_tree, or None to always point to the most recent commit + :param check: - If True, relatively expensive checks will be performed to verify - validity of the submodule. - :raise ValueError: If the commit's tree didn't contain the .gitmodules blob. + If True, relatively expensive checks will be performed to verify validity of + the submodule. + + :raise ValueError: + If the commit's tree didn't contain the ``.gitmodules`` blob. + :raise ValueError: If the parent commit didn't store this submodule under the current path. - :return: self + + :return: + self """ if commit is None: self._parent_commit = None @@ -1179,21 +1292,28 @@ def config_writer( self, index: Union["IndexFile", None] = None, write: bool = True ) -> SectionConstraint["SubmoduleConfigParser"]: """ - :return: A config writer instance allowing you to read and write the data - belonging to this submodule into the .gitmodules file. + :return: + A config writer instance allowing you to read and write the data belonging + to this submodule into the ``.gitmodules`` file. - :param index: If not None, an IndexFile instance which should be written. + :param index: + If not None, an IndexFile instance which should be written. Defaults to the index of the Submodule's parent repository. - :param write: If True, the index will be written each time a configuration - value changes. - :note: The parameters allow for a more efficient writing of the index, - as you can pass in a modified index on your own, prevent automatic writing, - and write yourself once the whole operation is complete. + :param write: + If True, the index will be written each time a configuration value changes. + + :note: + The parameters allow for a more efficient writing of the index, as you can + pass in a modified index on your own, prevent automatic writing, and write + yourself once the whole operation is complete. + + :raise ValueError: + If trying to get a writer on a parent_commit which does not match the + current head commit. - :raise ValueError: If trying to get a writer on a parent_commit which does not - match the current head commit. - :raise IOError: If the .gitmodules file/blob could not be read + :raise IOError: + If the ``.gitmodules`` file/blob could not be read. """ writer = self._config_parser_constrained(read_only=False) if index is not None: @@ -1208,22 +1328,24 @@ def rename(self, new_name: str) -> "Submodule": :note: This method takes care of renaming the submodule in various places, such as: - * $parent_git_dir / config - * $working_tree_dir / .gitmodules + * ``$parent_git_dir / config`` + * ``$working_tree_dir / .gitmodules`` * (git >= v1.8.0: move submodule repository to new name) - As .gitmodules will be changed, you would need to make a commit afterwards. The - changed .gitmodules file will already be added to the index. + As ``.gitmodules`` will be changed, you would need to make a commit afterwards. + The changed ``.gitmodules`` file will already be added to the index. - :return: This submodule instance + :return: + This :class:`Submodule` instance """ if self.name == new_name: return self # .git/config with self.repo.config_writer() as pw: - # As we ourselves didn't write anything about submodules into the parent .git/config, - # we will not require it to exist, and just ignore missing entries. + # As we ourselves didn't write anything about submodules into the parent + # .git/config, we will not require it to exist, and just ignore missing + # entries. if pw.has_section(sm_section(self.name)): pw.rename_section(sm_section(self.name), sm_section(new_name)) @@ -1260,8 +1382,9 @@ def module(self) -> "Repo": """ :return: Repo instance initialized from the repository at our submodule path - :raise InvalidGitRepositoryError: If a repository was not available. This could - also mean that it was not yet initialized. + :raise InvalidGitRepositoryError: + If a repository was not available. + This could also mean that it was not yet initialized. """ module_checkout_abspath = self.abspath try: @@ -1276,7 +1399,11 @@ def module(self) -> "Repo": # END handle exceptions def module_exists(self) -> bool: - """:return: True if our module exists and is a valid git repository. See module() method.""" + """ + :return: + True if our module exists and is a valid git repository. + See the :meth:`module` method. + """ try: self.module() return True @@ -1286,9 +1413,10 @@ def module_exists(self) -> bool: def exists(self) -> bool: """ - :return: True if the submodule exists, False otherwise. Please note that - a submodule may exist (in the .gitmodules file) even though its module - doesn't exist on disk. + :return: + True if the submodule exists, False otherwise. + Please note that a submodule may exist (in the ``.gitmodules`` file) even + though its module doesn't exist on disk. """ # Keep attributes for later, and restore them if we have no valid data. # This way we do not actually alter the state of the object. @@ -1299,7 +1427,8 @@ def exists(self) -> bool: loc[attr] = getattr(self, attr) # END if we have the attribute cache except (cp.NoSectionError, ValueError): - # On PY3, this can happen apparently... don't know why this doesn't happen on PY2. + # On PY3, this can happen apparently... don't know why this doesn't + # happen on PY2. pass # END for each attr self._clear_cache() @@ -1333,8 +1462,9 @@ def branch(self) -> "Head": @property def branch_path(self) -> PathLike: """ - :return: Complete relative path as string to the branch we would checkout - from the remote and track + :return: + Complete relative path as string to the branch we would checkout from the + remote and track """ return self._branch_path @@ -1344,8 +1474,8 @@ def branch_name(self) -> str: :return: The name of the branch, which is the shortest possible branch name """ - # Use an instance method, for this we create a temporary Head instance - # which uses a repository that is available at least (it makes no difference). + # Use an instance method, for this we create a temporary Head instance which + # uses a repository that is available at least (it makes no difference). return git.Head(self.repo, self._branch_path).name @property @@ -1420,7 +1550,8 @@ def iter_items( ) -> Iterator["Submodule"]: """ :return: - Iterator yielding Submodule instances available in the given repository + Iterator yielding :class:`Submodule` instances available in the given + repository """ try: pc = repo.commit(parent_commit) # Parent commit instance From 8344f442b4c002ec7f0a351c6a61d6d9ca084bce Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 21:19:11 -0500 Subject: [PATCH 0695/1392] Revise Repo.archive docstring I had missed this in b2b6f7c. --- git/repo/base.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index 1e58a24d0..71468bebc 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -1535,22 +1535,29 @@ def archive( ) -> Repo: """Archive the tree at the given revision. - :param ostream: file compatible stream object to which the archive will be written as bytes. + :param ostream: + File-compatible stream object to which the archive will be written as bytes. - :param treeish: is the treeish name/id, defaults to active branch. + :param treeish: + The treeish name/id, defaults to active branch. - :param prefix: is the optional prefix to prepend to each filename in the archive. + :param prefix: + The optional prefix to prepend to each filename in the archive. - :param kwargs: Additional arguments passed to git-archive: + :param kwargs: + Additional arguments passed to ``git archive``: - * Use the 'format' argument to define the kind of format. Use - specialized ostreams to write any format supported by python. - * You may specify the special **path** keyword, which may either be a repository-relative - path to a directory or file to place into the archive, or a list or tuple of multiple paths. + * Use the 'format' argument to define the kind of format. Use specialized + ostreams to write any format supported by Python. + * You may specify the special **path** keyword, which may either be a + repository-relative path to a directory or file to place into the archive, + or a list or tuple of multiple paths. - :raise GitCommandError: If something went wrong. + :raise GitCommandError: + If something went wrong. - :return: self + :return: + self """ if treeish is None: treeish = self.head.commit From 432ec72b759289cd8b8967847dd8bafcd6b78915 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 21:22:50 -0500 Subject: [PATCH 0696/1392] Fix another :raises: to :raise: Missed in bcc0c27. --- git/repo/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/repo/base.py b/git/repo/base.py index 71468bebc..19e951567 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -1020,7 +1020,7 @@ def ignored(self, *paths: PathLike) -> List[str]: def active_branch(self) -> Head: """The name of the currently active branch. - :raises TypeError: + :raise TypeError: If HEAD is detached. :return: From 5ca584444b1fe00d5c52341cd267588e64174d7f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 21:26:45 -0500 Subject: [PATCH 0697/1392] Fully qualify non-builtin exceptions in :raise: This expands exceptions from git.exc and gitdb.exc in :raise: sections of docstrings, to include those qualifiers. This serves a few purposes. The main benefits are: - Immediately clarify which exceptions are not built-in ones, as not all readers are necessarily familiar with all built-in exceptions. - Distinguish exceptions defined in GitPython (in git.exc) from those defined in gitdb (in gitdb.exc). Although the gitdb exceptions GitPython uses are imported into git.exc and listed in __all__, they are still not introduced in GitPython, which is relevant for (a) preventing confusion, so developers don't think they accidentally imported a different same-named exception, (b) understanding why they do not have documentation in the generated GitPython docs. It also has these minor benefits: - May help sphinx-autodoc find them. In practice I think this is unlikely to be necessary. - Most other references, including references to classes, within docstrings in the git module, are now fully qualified, when not defined/introduced in the same module. So this is consistent with that. This consistency might be more than a minor benefit if there were a committment to keep most such refernces fully qualified, but this really should not be a goal: especially where Sphinx autodoc is guaranteed to be able to resolve them, shortening (in some cases, re-shortening) the references in the future may lead to better docstring readability. --- git/cmd.py | 4 ++-- git/db.py | 8 ++++---- git/index/base.py | 6 +++--- git/index/fun.py | 2 +- git/objects/submodule/base.py | 6 +++--- git/remote.py | 2 +- git/repo/base.py | 10 +++++----- git/repo/fun.py | 4 ++-- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index cab089746..997f9d41e 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -699,7 +699,7 @@ def wait(self, stderr: Union[None, str, bytes] = b"") -> int: May deadlock if output or error pipes are used and not handled separately. - :raise GitCommandError: + :raise git.exc.GitCommandError: If the return status is not 0. """ if stderr is None: @@ -1091,7 +1091,7 @@ def execute( Note that git is executed with ``LC_MESSAGES="C"`` to ensure consistent output regardless of system language. - :raise GitCommandError: + :raise git.exc.GitCommandError: :note: If you add additional keyword arguments to the signature of this method, diff --git a/git/db.py b/git/db.py index dff59f47a..eb7f758da 100644 --- a/git/db.py +++ b/git/db.py @@ -56,13 +56,13 @@ def partial_to_complete_sha_hex(self, partial_hexsha: str) -> bytes: """ :return: Full binary 20 byte sha from the given partial hexsha - :raise AmbiguousObjectName: + :raise gitdb.exc.AmbiguousObjectName: - :raise BadObject: + :raise gitdb.exc.BadObject: :note: - Currently we only raise :class:`BadObject` as git does not communicate - ambiguous objects separately. + Currently we only raise :class:`~gitdb.exc.BadObject` as git does not + communicate ambiguous objects separately. """ try: hexsha, _typename, _size = self._git.get_object_header(partial_hexsha) diff --git a/git/index/base.py b/git/index/base.py index 31186b203..5c738cdf2 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -284,7 +284,7 @@ def merge_tree(self, rhs: Treeish, base: Union[None, Treeish] = None) -> "IndexF self (containing the merge and possibly unmerged entries in case of conflicts) - :raise GitCommandError: + :raise git.exc.GitCommandError: If there is a merge conflict. The error will be raised at the first conflicting path. If you want to have proper merge resolution to be done by yourself, you have to commit the changed index (or make a valid tree from @@ -624,7 +624,7 @@ def write_tree(self) -> Tree: :raise ValueError: If there are no entries in the cache. - :raise UnmergedEntriesError: + :raise git.exc.UnmergedEntriesError: """ # We obtain no lock as we just flush our contents to disk as tree. # If we are a new index, the entries access will load our data accordingly. @@ -1077,7 +1077,7 @@ def move( :raise ValueError: If only one item was given. - :raise GitCommandError: + :raise git.exc.GitCommandError: If git could not handle your request. """ args = [] diff --git a/git/index/fun.py b/git/index/fun.py index a60a3b809..1d4ca9b93 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -88,7 +88,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: :param args: Arguments passed to hook file. - :raise HookExecutionError: + :raise git.exc.HookExecutionError: """ hp = hook_path(name, index.repo.git_dir) if not os.access(hp, os.X_OK): diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 70a021c54..159403f24 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1106,7 +1106,7 @@ def remove( Doesn't work atomically, as failure to remove any part of the submodule will leave an inconsistent state. - :raise InvalidGitRepositoryError: + :raise git.exc.InvalidGitRepositoryError: Thrown if the repository cannot be deleted. :raise OSError: @@ -1382,7 +1382,7 @@ def module(self) -> "Repo": """ :return: Repo instance initialized from the repository at our submodule path - :raise InvalidGitRepositoryError: + :raise git.exc.InvalidGitRepositoryError: If a repository was not available. This could also mean that it was not yet initialized. """ @@ -1454,7 +1454,7 @@ def branch(self) -> "Head": :return: The branch instance that we are to checkout - :raise InvalidGitRepositoryError: + :raise git.exc.InvalidGitRepositoryError: If our module is not yet checked out. """ return mkhead(self.module(), self._branch_path) diff --git a/git/remote.py b/git/remote.py index 5bee9edf5..249a6b7bf 100644 --- a/git/remote.py +++ b/git/remote.py @@ -801,7 +801,7 @@ def create(cls, repo: "Repo", name: str, url: str, allow_unsafe_protocols: bool :return: New :class:`Remote` instance - :raise GitCommandError: + :raise git.exc.GitCommandError: In case an origin with that name already exists. """ scmd = "add" diff --git a/git/repo/base.py b/git/repo/base.py index 19e951567..7b3e514c1 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -202,12 +202,12 @@ def __init__( Please note that this was the default behaviour in older versions of GitPython, which is considered a bug though. - :raise InvalidGitRepositoryError: + :raise git.exc.InvalidGitRepositoryError: - :raise NoSuchPathError: + :raise git.exc.NoSuchPathError: :return: - git.Repo + :class:`Repo` """ epath = path or os.getenv("GIT_DIR") @@ -893,7 +893,7 @@ def _set_alternates(self, alts: List[str]) -> None: The array of string paths representing the alternates at which git should look for objects, i.e. ``/home/user/repo/.git/objects``. - :raise NoSuchPathError: + :raise git.exc.NoSuchPathError: :note: The method does not check for the existence of the paths in `alts`, as the @@ -1553,7 +1553,7 @@ def archive( repository-relative path to a directory or file to place into the archive, or a list or tuple of multiple paths. - :raise GitCommandError: + :raise git.exc.GitCommandError: If something went wrong. :return: diff --git a/git/repo/fun.py b/git/repo/fun.py index fa2cebb67..4936f1773 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -59,7 +59,7 @@ def touch(filename: str) -> str: def is_git_dir(d: "PathLike") -> bool: """This is taken from the git setup.c:is_git_directory function. - :raise WorkTreeRepositoryUnsupported: + :raise git.exc.WorkTreeRepositoryUnsupported: If it sees a worktree directory. It's quite hacky to do that here, but at least clearly indicates that we don't support it. There is the unlikely danger to throw if we see directories which just look like a worktree dir, but are none. @@ -234,7 +234,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: ``git rev-parse``-compatible revision specification as string. Please see http://www.kernel.org/pub/software/scm/git/docs/git-rev-parse.html for details. - :raise BadObject: + :raise gitdb.exc.BadObject: If the given revision could not be found. :raise ValueError: From e6768ec9a06c295808ea1b4b7ed91e57fb4a2bf3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 27 Feb 2024 21:50:38 -0500 Subject: [PATCH 0698/1392] Improve Git.execute docstring formatting re: max_chunk_size --- git/cmd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 997f9d41e..4bd53ea0f 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1067,9 +1067,9 @@ def execute( A dictionary of environment variables to be passed to :class:`subprocess.Popen`. :param max_chunk_size: - Maximum number of bytes in one chunk of data passed to the output_stream in - one invocation of write() method. If the given number is not positive then - the default value is used. + Maximum number of bytes in one chunk of data passed to the `output_stream` + in one invocation of its ``write()`` method. If the given number is not + positive then the default value is used. :param strip_newline_in_stdout: Whether to strip the trailing ``\n`` of the command stdout. From cd61eb4f7a5c9cd1c09b362335577d87a441e2c1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 17:23:04 -0500 Subject: [PATCH 0699/1392] Further revise docstrings in second-level modules Some stuff unintentionally missed in 1cd73ba and some subsequent commits, plus a few further things intentionally omitted there, such as converting True, False, and None literals, to double backticked form (rather than bare or single-backticked). --- git/__init__.py | 9 +++-- git/cmd.py | 103 +++++++++++++++++++++++++----------------------- git/config.py | 46 +++++++++++---------- git/diff.py | 17 ++++---- git/exc.py | 4 +- git/remote.py | 37 ++++++++--------- git/util.py | 8 ++-- 7 files changed, 119 insertions(+), 105 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index bc97a6eff..6fc6110f4 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -129,12 +129,13 @@ def refresh(path: Optional[PathLike] = None) -> None: :note: The *path* parameter is usually omitted and cannot be used to specify a custom command whose location is looked up in a path search on each call. See - :meth:`Git.refresh` for details on how to achieve this. + :meth:`Git.refresh ` for details on how to achieve this. :note: - This calls :meth:`Git.refresh` and sets other global configuration according to - the effect of doing so. As such, this function should usually be used instead of - using :meth:`Git.refresh` or :meth:`FetchInfo.refresh` directly. + This calls :meth:`Git.refresh ` and sets other global + configuration according to the effect of doing so. As such, this function should + usually be used instead of using :meth:`Git.refresh ` or + :meth:`FetchInfo.refresh ` directly. :note: This function is called automatically, with no arguments, at import time. diff --git a/git/cmd.py b/git/cmd.py index 4bd53ea0f..d0c76eafe 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -112,25 +112,25 @@ def handle_process_output( This function returns once the finalizer returns. :param process: - :class:`subprocess.Popen` instance + :class:`subprocess.Popen` instance. :param stdout_handler: - f(stdout_line_string), or None + f(stdout_line_string), or ``None``. :param stderr_handler: - f(stderr_line_string), or None + f(stderr_line_string), or ``None``. :param finalizer: - f(proc) - wait for proc to finish + f(proc) - wait for proc to finish. :param decode_streams: Assume stdout/stderr streams are binary and decode them before pushing their contents to handlers. - Set this to False if ``universal_newlines == True`` (then streams are in text - mode) or if decoding must happen later (i.e. for :class:`git.diff.Diff`s). + Set this to ``False`` if ``universal_newlines == True`` (then streams are in + text mode) or if decoding must happen later (i.e. for :class:`~git.diff.Diff`s). :param kill_after_timeout: - float or None, Default = None + :class:`float` or ``None``, Default = ``None`` To specify a timeout in seconds for the git command, after which the process should be killed. @@ -236,7 +236,7 @@ def _safer_popen_windows( itself be searched automatically by the shell. This wrapper covers all those cases. :note: - This currently works by setting the ``NoDefaultCurrentDirectoryInExePath`` + This currently works by setting the :envvar:`NoDefaultCurrentDirectoryInExePath` environment variable during subprocess creation. It also takes care of passing Windows-specific process creation flags, but that is unrelated to path search. @@ -311,8 +311,8 @@ class Git: Debugging: - * Set the ``GIT_PYTHON_TRACE`` environment variable print each invocation of the - command to stdout. + * Set the :envvar:`GIT_PYTHON_TRACE` environment variable to print each invocation + of the command to stdout. * Set its value to ``full`` to see details about the returned values. """ @@ -351,7 +351,7 @@ def __setstate__(self, d: Dict[str, Any]) -> None: """Enables debugging of GitPython's git commands.""" USE_SHELL = False - """Deprecated. If set to True, a shell will be used when executing git commands. + """Deprecated. If set to ``True``, a shell will be used when executing git commands. Prior to GitPython 2.0.8, this had a narrow purpose in suppressing console windows in graphical Windows applications. In 2.0.8 and higher, it provides no benefit, as @@ -401,30 +401,32 @@ def refresh(cls, path: Union[None, PathLike] = None) -> bool: There are three different ways to specify the command that refreshing causes to be used for git: - 1. Pass no `path` argument and do not set the ``GIT_PYTHON_GIT_EXECUTABLE`` - environment variable. The command name ``git`` is used. It is looked up - in a path search by the system, in each command run (roughly similar to - how git is found when running ``git`` commands manually). This is usually - the desired behavior. + 1. Pass no `path` argument and do not set the + :envvar:`GIT_PYTHON_GIT_EXECUTABLE` environment variable. The command + name ``git`` is used. It is looked up in a path search by the system, in + each command run (roughly similar to how git is found when running + ``git`` commands manually). This is usually the desired behavior. - 2. Pass no `path` argument but set the ``GIT_PYTHON_GIT_EXECUTABLE`` + 2. Pass no `path` argument but set the :envvar:`GIT_PYTHON_GIT_EXECUTABLE` environment variable. The command given as the value of that variable is used. This may be a simple command or an arbitrary path. It is looked up - in each command run. Setting ``GIT_PYTHON_GIT_EXECUTABLE`` to ``git`` has - the same effect as not setting it. + in each command run. Setting :envvar:`GIT_PYTHON_GIT_EXECUTABLE` to + ``git`` has the same effect as not setting it. 3. Pass a `path` argument. This path, if not absolute, is immediately resolved, relative to the current directory. This resolution occurs at the time of the refresh. When git commands are run, they are run using that previously resolved path. If a `path` argument is passed, the - ``GIT_PYTHON_GIT_EXECUTABLE`` environment variable is not consulted. + :envvar:`GIT_PYTHON_GIT_EXECUTABLE` environment variable is not + consulted. :note: Refreshing always sets the :attr:`Git.GIT_PYTHON_GIT_EXECUTABLE` class attribute, which can be read on the :class:`Git` class or any of its instances to check what command is used to run git. This attribute should - not be confused with the related ``GIT_PYTHON_GIT_EXECUTABLE`` environment - variable. The class attribute is set no matter how refreshing is performed. + not be confused with the related :envvar:`GIT_PYTHON_GIT_EXECUTABLE` + environment variable. The class attribute is set no matter how refreshing is + performed. """ # Discern which path to refresh with. if path is not None: @@ -571,9 +573,9 @@ def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> str: def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> PathLike: """Remove any backslashes from URLs to be written in config files. - Windows might create config files containing paths with backslashes, - but git stops liking them as it will escape the backslashes. Hence we - undo the escaping just to be sure. + Windows might create config files containing paths with backslashes, but git + stops liking them as it will escape the backslashes. Hence we undo the escaping + just to be sure. """ if is_cygwin is None: is_cygwin = cls.is_cygwin() @@ -638,8 +640,8 @@ class AutoInterrupt: __slots__ = ("proc", "args", "status") - # If this is non-zero it will override any status code during - # _terminate, used to prevent race conditions in testing. + # If this is non-zero it will override any status code during _terminate, used + # to prevent race conditions in testing. _status_code_if_terminate: int = 0 def __init__(self, proc: Union[None, subprocess.Popen], args: Any) -> None: @@ -844,7 +846,7 @@ def __init__(self, working_dir: Union[None, PathLike] = None): """Initialize this instance with: :param working_dir: - Git directory we should work in. If None, we always work in the current + Git directory we should work in. If ``None``, we always work in the current directory as returned by :func:`os.getcwd`. This is meant to be the working tree directory if available, or the ``.git`` directory in case of bare repositories. @@ -1022,8 +1024,8 @@ def execute( This feature only has any effect if `as_process` is False. :param stdout_as_string: - If False, the command's standard output will be bytes. Otherwise, it will be - decoded into a string using the default encoding (usually UTF-8). + If ``False``, the command's standard output will be bytes. Otherwise, it + will be decoded into a string using the default encoding (usually UTF-8). The latter can fail, if the output contains binary data. :param kill_after_timeout: @@ -1045,15 +1047,16 @@ def execute( is manually removed. :param with_stdout: - If True, default True, we open stdout on the created process. + If ``True``, default ``True``, we open stdout on the created process. :param universal_newlines: - If True, pipes will be opened as text, and lines are split at all known line - endings. + If ``True``, pipes will be opened as text, and lines are split at all known + line endings. :param shell: - Whether to invoke commands through a shell (see `Popen(..., shell=True)`). - If this is not `None`, it overrides :attr:`USE_SHELL`. + Whether to invoke commands through a shell + (see :class:`Popen(..., shell=True) `). + If this is not ``None``, it overrides :attr:`USE_SHELL`. Passing ``shell=True`` to this or any other GitPython function should be avoided, as it is unsafe under most circumstances. This is because it is @@ -1064,7 +1067,8 @@ def execute( issues. :param env: - A dictionary of environment variables to be passed to :class:`subprocess.Popen`. + A dictionary of environment variables to be passed to + :class:`subprocess.Popen`. :param max_chunk_size: Maximum number of bytes in one chunk of data passed to the `output_stream` @@ -1083,7 +1087,7 @@ def execute( * str(output) if extended_output = False (Default) * tuple(int(status), str(stdout), str(stderr)) if extended_output = True - If output_stream is True, the stdout value will be your output stream: + If `output_stream` is ``True``, the stdout value will be your output stream: * output_stream if extended_output = False * tuple(int(status), output_stream, str(stderr)) if extended_output = True @@ -1288,7 +1292,7 @@ def update_environment(self, **kwargs: Any) -> Dict[str, Union[str, None]]: self.update_environment(**old_env) :param kwargs: - Environment variables to use for git processes + Environment variables to use for git processes. :return: Dict that maps environment variables to their old values @@ -1367,8 +1371,8 @@ def __call__(self, **kwargs: Any) -> "Git": :param kwargs: A dict of keyword arguments. - These arguments are passed as in :meth:`_call_process`, but will be - passed to the git command rather than the subcommand. + These arguments are passed as in :meth:`_call_process`, but will be passed + to the git command rather than the subcommand. Examples:: @@ -1409,16 +1413,16 @@ def _call_process( as in ``ls_files`` to call ``ls-files``. :param args: - The list of arguments. If None is included, it will be pruned. - This allows your commands to call git more conveniently, as None is realized - as non-existent. + The list of arguments. If ``None`` is included, it will be pruned. + This allows your commands to call git more conveniently, as ``None`` is + realized as non-existent. :param kwargs: Contains key-values for the following: - The :meth:`execute()` kwds, as listed in :var:`execute_kwargs`. - "Command options" to be converted by :meth:`transform_kwargs`. - - The `'insert_kwargs_after'` key which its value must match one of ``*args``. + - The ``insert_kwargs_after`` key which its value must match one of ``*args``. It also contains any command options, to be appended after the matched arg. @@ -1431,9 +1435,9 @@ def _call_process( git rev-list max-count 10 --header master :return: - Same as :meth:`execute`. - If no args are given, used :meth:`execute`'s default (especially - ``as_process = False``, ``stdout_as_string = True``) and return str. + Same as :meth:`execute`. If no args are given, used :meth:`execute`'s + default (especially ``as_process = False``, ``stdout_as_string = True``) and + return :class:`str`. """ # Handle optional arguments prior to calling transform_kwargs. # Otherwise these'll end up in args, which is bad. @@ -1480,10 +1484,11 @@ def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: :param header_line: type_string size_as_int - :return: (hex_sha, type_string, size_as_int) + :return: + (hex_sha, type_string, size_as_int) :raise ValueError: - If the header contains indication for an error due to incorrect input sha + If the header contains indication for an error due to incorrect input sha. """ tokens = header_line.split() if len(tokens) != 3: diff --git a/git/config.py b/git/config.py index b5cff01cd..ff5c9d564 100644 --- a/git/config.py +++ b/git/config.py @@ -186,8 +186,8 @@ def config(self) -> T_ConfigParser: return self._config def release(self) -> None: - """Equivalent to GitConfigParser.release(), which is called on our underlying - parser instance.""" + """Equivalent to :meth:`GitConfigParser.release`, which is called on our + underlying parser instance.""" return self._config.release() def __enter__(self) -> "SectionConstraint[T_ConfigParser]": @@ -292,7 +292,7 @@ class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): t_lock = LockFile """The lock type determines the type of lock to use in new configuration readers. - They must be compatible to the LockFile interface. + They must be compatible to the :class:`~git.util.LockFile` interface. A suitable alternative would be the :class:`~git.util.BlockingLockFile`. """ @@ -326,14 +326,15 @@ def __init__( A file path or file object, or a sequence of possibly more than one of them. :param read_only: - If True, the ConfigParser may only read the data, but not change it. - If False, only a single file path or file object may be given. We will write - back the changes when they happen, or when the ConfigParser is released. - This will not happen if other configuration files have been included. + If ``True``, the ConfigParser may only read the data, but not change it. + If ``False``, only a single file path or file object may be given. We will + write back the changes when they happen, or when the ConfigParser is + released. This will not happen if other configuration files have been + included. :param merge_includes: - If True, we will read files mentioned in ``[include]`` sections and merge - their contents into ours. This makes it impossible to write back an + If ``True``, we will read files mentioned in ``[include]`` sections and + merge their contents into ours. This makes it impossible to write back an individual configuration file. Thus, if you want to modify a single configuration file, turn this off to leave the original dataset unaltered when reading it. @@ -347,7 +348,8 @@ def __init__( self._defaults: _OMD self._sections: _OMD # type: ignore # mypy/typeshed bug? - # Used in Python 3. Needs to stay in sync with sections for underlying implementation to work. + # Used in Python 3. Needs to stay in sync with sections for underlying + # implementation to work. if not hasattr(self, "_proxies"): self._proxies = self._dict() @@ -593,7 +595,7 @@ def read(self) -> None: # type: ignore[override] Nothing :raise IOError: - If a file cannot be handled + If a file cannot be handled. """ if self._is_initialized: return @@ -712,7 +714,8 @@ def write(self) -> None: """Write changes to our file, if there are changes at all. :raise IOError: - If this is a read-only writer instance or if we could not obtain a file lock + If this is a read-only writer instance or if we could not obtain a file + lock. """ self._assure_writable("write") if not self._dirty: @@ -778,8 +781,8 @@ def get_value( specified is returned. :param default: - If not None, the given default value will be returned in case the option did - not exist + If not ``None``, the given default value will be returned in case the option + did not exist. :return: A properly typed value, either int, float or string @@ -809,8 +812,8 @@ def get_values( returned. :param default: - If not None, a list containing the given default value will be returned in - case the option did not exist. + If not ``None``, a list containing the given default value will be returned + in case the option did not exist. :return: A list of properly typed values, either int, float or string @@ -868,7 +871,7 @@ def set_value(self, section: str, option: str, value: Union[str, bytes, int, flo """Set the given option in section to the given value. This will create the section if required, and will not throw as opposed to the - default ConfigParser 'set' method. + default ConfigParser ``set`` method. :param section: Name of the section in which the option resides or should reside. @@ -893,9 +896,9 @@ def add_value(self, section: str, option: str, value: Union[str, bytes, int, flo """Add a value for the given option in section. This will create the section if required, and will not throw as opposed to the - default ConfigParser 'set' method. The value becomes the new value of the option - as returned by 'get_value', and appends to the list of values returned by - 'get_values'. + default ConfigParser ``set`` method. The value becomes the new value of the + option as returned by :meth:`get_value`, and appends to the list of values + returned by :meth:`get_values`. :param section: Name of the section in which the option resides or should reside. @@ -923,7 +926,8 @@ def rename_section(self, section: str, new_name: str) -> "GitConfigParser": * ``section`` doesn't exist. * A section with ``new_name`` does already exist. - :return: This instance + :return: + This instance """ if not self.has_section(section): raise ValueError("Source section '%s' doesn't exist" % section) diff --git a/git/diff.py b/git/diff.py index 7205136d0..3c760651b 100644 --- a/git/diff.py +++ b/git/diff.py @@ -118,7 +118,7 @@ def diff( :param other: This the item to compare us with. - * If None, we will be compared to the working tree. + * If ``None``, we will be compared to the working tree. * If :class:`~git.index.base.Treeish`, it will be compared against the respective tree. * If :class:`~Diffable.Index`, it will be compared against the index. @@ -131,7 +131,7 @@ def diff( include at least one of the given path or paths. :param create_patch: - If True, the returned :class:`Diff` contains a detailed patch that if + If ``True``, the returned :class:`Diff` contains a detailed patch that if applied makes the self to other. Patches are somewhat costly as blobs have to be read and diffed. @@ -139,7 +139,8 @@ def diff( Additional arguments passed to ``git diff``, such as ``R=True`` to swap both sides of the diff. - :return: git.DiffIndex + :return: + :class:`DiffIndex` :note: On a bare repository, `other` needs to be provided as @@ -262,7 +263,7 @@ class Diff: Diffs keep information about the changed blob objects, the file mode, renames, deletions and new files. - There are a few cases where None has to be expected as member variable value: + There are a few cases where ``None`` has to be expected as member variable value: New File:: @@ -468,7 +469,8 @@ def rename_to(self) -> Optional[str]: @property def renamed(self) -> bool: """ - :return: True if the blob of our diff has been renamed + :return: + ``True`` if the blob of our diff has been renamed :note: This property is deprecated. @@ -499,11 +501,12 @@ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoIn """Create a new :class:`DiffIndex` from the given process output which must be in patch format. - :param repo: The repository we are operating on. + :param repo: + The repository we are operating on. :param proc: ``git diff`` process to read from - (supports :class:`Git.AutoInterrupt` wrapper). + (supports :class:`Git.AutoInterrupt ` wrapper). :return: :class:`DiffIndex` diff --git a/git/exc.py b/git/exc.py index 276d79e42..9f6462b39 100644 --- a/git/exc.py +++ b/git/exc.py @@ -136,8 +136,8 @@ def __str__(self) -> str: class GitCommandNotFound(CommandError): - """Thrown if we cannot find the `git` executable in the ``PATH`` or at the path - given by the ``GIT_PYTHON_GIT_EXECUTABLE`` environment variable.""" + """Thrown if we cannot find the ``git`` executable in the :envvar:`PATH` or at the + path given by the :envvar:`GIT_PYTHON_GIT_EXECUTABLE` environment variable.""" def __init__(self, command: Union[List[str], Tuple[str], str], cause: Union[str, Exception]) -> None: super().__init__(command, cause) diff --git a/git/remote.py b/git/remote.py index 249a6b7bf..265f1901f 100644 --- a/git/remote.py +++ b/git/remote.py @@ -125,8 +125,7 @@ def to_progress_instance( class PushInfo(IterableObj): - """ - Carries information about the result of a push operation of a single head:: + """Carries information about the result of a push operation of a single head:: info = remote.push()[0] info.flags # bitflags providing more information about the result @@ -222,8 +221,8 @@ def remote_ref(self) -> Union[RemoteReference, TagReference]: @classmethod def _from_line(cls, remote: "Remote", line: str) -> "PushInfo": - """Create a new PushInfo instance as parsed from line which is expected to be like - refs/heads/master:refs/heads/master 05d2687..1d0568e as bytes.""" + """Create a new :class:`PushInfo` instance as parsed from line which is expected + to be like refs/heads/master:refs/heads/master 05d2687..1d0568e as bytes.""" control_character, from_to, summary = line.split("\t", 3) flags = 0 @@ -279,7 +278,7 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> NoReturn: # -> class PushInfoList(IterableList[PushInfo]): - """IterableList of :class:`PushInfo` objects.""" + """:class:`~git.util.IterableList` of :class:`PushInfo` objects.""" def __new__(cls) -> "PushInfoList": return cast(PushInfoList, IterableList.__new__(cls, "push_infos")) @@ -295,8 +294,7 @@ def raise_if_error(self) -> None: class FetchInfo(IterableObj): - """ - Carries information about the results of a fetch operation of a single head:: + """Carries information about the results of a fetch operation of a single head:: info = remote.fetch()[0] info.ref # Symbolic Reference or RemoteReference to the changed @@ -559,7 +557,7 @@ def __init__(self, repo: "Repo", name: str) -> None: The repository we are a remote of. :param name: - The name of the remote, e.g. 'origin'. + The name of the remote, e.g. ``origin``. """ self.repo = repo self.name = name @@ -611,7 +609,7 @@ def __hash__(self) -> int: def exists(self) -> bool: """ :return: - True if this is a valid, existing remote. + ``True`` if this is a valid, existing remote. Valid remotes have an entry in the repository's configuration. """ try: @@ -683,7 +681,7 @@ def add_url(self, url: str, allow_unsafe_protocols: bool = False, **kwargs: Any) return self.set_url(url, add=True, allow_unsafe_protocols=allow_unsafe_protocols) def delete_url(self, url: str, **kwargs: Any) -> "Remote": - """Deletes a new url on current remote (special case of ``git remote set-url``) + """Deletes a new url on current remote (special case of ``git remote set-url``). This command deletes new URLs to a given remote, making it possible to have multiple URLs for a single remote. @@ -733,7 +731,7 @@ def urls(self) -> Iterator[str]: def refs(self) -> IterableList[RemoteReference]: """ :return: - :class:`~git.util.IterableList` of :class:`git.refs.remote.RemoteReference` + :class:`~git.util.IterableList` of :class:`~git.refs.remote.RemoteReference` objects. It is prefixed, allowing you to omit the remote path portion, e.g.:: @@ -748,7 +746,7 @@ def refs(self) -> IterableList[RemoteReference]: def stale_refs(self) -> IterableList[Reference]: """ :return: - :class:`~git.util.IterableList` of :class:`git.refs.remote.RemoteReference` + :class:`~git.util.IterableList` of :class:`~git.refs.remote.RemoteReference` objects that do not have a corresponding head in the remote reference anymore as they have been deleted on the remote side, but are still available locally. @@ -1019,13 +1017,15 @@ def fetch( supplying a list rather than a string for 'refspec' will make use of this facility. - :param progress: See :meth:`push` method. + :param progress: + See :meth:`push` method. - :param verbose: Boolean for verbose output. + :param verbose: + Boolean for verbose output. :param kill_after_timeout: To specify a timeout in seconds for the git command, after which the process - should be killed. It is set to None by default. + should be killed. It is set to ``None`` by default. :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ``ext``. @@ -1101,7 +1101,7 @@ def pull( Additional arguments to be passed to ``git pull``. :return: - Please see :meth:`fetch` method + Please see :meth:`fetch` method. """ if refspec is None: # No argument refspec, then ensure the repo's config has a fetch refspec. @@ -1141,7 +1141,7 @@ def push( :param progress: Can take one of many value types: - * None, to discard progress information. + * ``None``, to discard progress information. * A function (callable) that is called with the progress information. Signature: ``progress(op_code, cur_count, max_count=None, message='')``. See :meth:`RemoteProgress.update ` for a @@ -1163,7 +1163,8 @@ def push( :param allow_unsafe_options: Allow unsafe options to be used, like ``--receive-pack``. - :param kwargs: Additional arguments to be passed to ``git push``. + :param kwargs: + Additional arguments to be passed to ``git push``. :return: A :class:`PushInfoList` object, where each list member represents an diff --git a/git/util.py b/git/util.py index 4929affeb..e7dd94a08 100644 --- a/git/util.py +++ b/git/util.py @@ -112,7 +112,7 @@ def _read_win_env_flag(name: str, default: bool) -> bool: :return: On Windows, the flag, or the `default` value if absent or ambiguous. - On all other operating systems, False. + On all other operating systems, ``False``. :note: This only accesses the environment on Windows. @@ -309,11 +309,11 @@ def assure_directory_exists(path: PathLike, is_file: bool = False) -> bool: """Make sure that the directory pointed to by path exists. :param is_file: - If True, `path` is assumed to be a file and handled correctly. + If ``True``, `path` is assumed to be a file and handled correctly. Otherwise it must be a directory. :return: - True if the directory was created, False if it already existed. + ``True`` if the directory was created, ``False`` if it already existed. """ if is_file: path = osp.dirname(path) @@ -734,7 +734,7 @@ def update( Current absolute count of items. :param max_count: - The maximum count of items we expect. It may be None in case there is no + The maximum count of items we expect. It may be ``None`` in case there is no maximum number of items or if it is (yet) unknown. :param message: From fc1762b94ec9c8e22e0eb3b817604d15153f01b2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 17:26:38 -0500 Subject: [PATCH 0700/1392] Undo a couple minor black-incompatible changes Two of the docstrings in git.remote need to have a newline before the conceptually "first" line in order for black to accept the indentation of the subsequent text, which appears intentional and works fine as reStructuredText. Otherwise black dedents the code block, which affects rendering and is also less readable. --- git/remote.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/git/remote.py b/git/remote.py index 265f1901f..3d5b8fdc4 100644 --- a/git/remote.py +++ b/git/remote.py @@ -125,7 +125,8 @@ def to_progress_instance( class PushInfo(IterableObj): - """Carries information about the result of a push operation of a single head:: + """ + Carries information about the result of a push operation of a single head:: info = remote.push()[0] info.flags # bitflags providing more information about the result @@ -294,7 +295,8 @@ def raise_if_error(self) -> None: class FetchInfo(IterableObj): - """Carries information about the results of a fetch operation of a single head:: + """ + Carries information about the results of a fetch operation of a single head:: info = remote.fetch()[0] info.ref # Symbolic Reference or RemoteReference to the changed From 1b25a13a36ec641ffd63a6f1a97c3bdf8f6d28de Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 17:30:46 -0500 Subject: [PATCH 0701/1392] Further revise docstrings within git.index --- git/index/base.py | 147 ++++++++++++++++++++++++---------------------- git/index/fun.py | 4 +- git/index/typ.py | 36 +++++++----- 3 files changed, 99 insertions(+), 88 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 5c738cdf2..249144d54 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -3,8 +3,8 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Module containing IndexFile, an Index implementation facilitating all kinds of index -manipulations such as querying and merging.""" +"""Module containing :class:`IndexFile`, an Index implementation facilitating all kinds +of index manipulations such as querying and merging.""" import contextlib import datetime @@ -125,9 +125,9 @@ class IndexFile(LazyMixin, git_diff.Diffable, Serializable): git command function calls wherever possible. This provides custom merging facilities allowing to merge without actually changing - your index or your working tree. This way you can perform own test-merges based on - the index only without having to deal with the working copy. This is useful in case - of partial working trees. + your index or your working tree. This way you can perform your own test merges based + on the index only without having to deal with the working copy. This is useful in + case of partial working trees. Entries: @@ -211,7 +211,7 @@ def _deserialize(self, stream: IO) -> "IndexFile": return self def _entries_sorted(self) -> List[IndexEntry]: - """:return: list of entries, in a sorted fashion, first by path, then by stage""" + """:return: List of entries, in a sorted fashion, first by path, then by stage""" return sorted(self.entries.values(), key=lambda e: (e.path, e.stage)) def _serialize(self, stream: IO, ignore_extension_data: bool = False) -> "IndexFile": @@ -232,17 +232,17 @@ def write( """Write the current state to our file path or to the given one. :param file_path: - If None, we will write to our stored file path from which we have been + If ``None``, we will write to our stored file path from which we have been initialized. Otherwise we write to the given file path. Please note that - this will change the file_path of this index to the one you gave. + this will change the `file_path` of this index to the one you gave. :param ignore_extension_data: - If True, the TREE type extension data read in the index will not be written - to disk. NOTE that no extension data is actually written. - Use this if you have altered the index and would like to use git-write-tree - afterwards to create a tree representing your written changes. - If this data is present in the written index, git-write-tree will instead - write the stored/cached tree. + If ``True``, the TREE type extension data read in the index will not be + written to disk. NOTE that no extension data is actually written. + Use this if you have altered the index and would like to use + ``git write-tree`` afterwards to create a tree representing your written + changes. If this data is present in the written index, ``git write-tree`` + will instead write the stored/cached tree. Alternatively, use :meth:`write_tree` to handle this case automatically. """ # Make sure we have our entries read before getting a write lock. @@ -479,8 +479,8 @@ def _write_path_to_stdin( stdout string :param read_from_stdout: - If True, proc.stdout will be read after the item was sent to stdin. In that - case, it will return None. + If ``True``, ``proc.stdout`` will be read after the item was sent to stdin. + In that case, it will return ``None``. :note: There is a bug in git-update-index that prevents it from sending reports @@ -516,12 +516,13 @@ def iter_blobs( ) -> Iterator[Tuple[StageType, Blob]]: """ :return: - Iterator yielding tuples of Blob objects and stages, tuple(stage, Blob). + Iterator yielding tuples of :class:`~git.objects.blob.Blob` objects and + stages, tuple(stage, Blob). :param predicate: - Function(t) returning True if tuple(stage, Blob) should be yielded by the - iterator. A default filter, the `~git.index.typ.BlobFilter`, allows you to - yield blobs only if they match a given list of paths. + Function(t) returning ``True`` if tuple(stage, Blob) should be yielded by + the iterator. A default filter, the `~git.index.typ.BlobFilter`, allows you + to yield blobs only if they match a given list of paths. """ for entry in self.entries.values(): blob = entry.to_blob(self.repo) @@ -534,8 +535,8 @@ def iter_blobs( def unmerged_blobs(self) -> Dict[PathLike, List[Tuple[StageType, Blob]]]: """ :return: - Dict(path : list( tuple( stage, Blob, ...))), being a dictionary associating - a path in the index with a list containing sorted stage/blob pairs. + Dict(path : list(tuple(stage, Blob, ...))), being a dictionary associating a + path in the index with a list containing sorted stage/blob pairs. :note: Blobs that have been removed in one side simply do not exist in the given @@ -562,8 +563,8 @@ def resolve_blobs(self, iter_blobs: Iterator[Blob]) -> "IndexFile": This will effectively remove the index entries of the respective path at all non-null stages and add the given blob as new stage null blob. - For each path there may only be one blob, otherwise a ValueError will be raised - claiming the path is already at stage 0. + For each path there may only be one blob, otherwise a :class:`ValueError` will + be raised claiming the path is already at stage 0. :raise ValueError: If one of the blobs already existed at stage 0. @@ -603,7 +604,8 @@ def update(self) -> "IndexFile": This is a possibly dangerous operations as it will discard your changes to :attr:`index.entries `. - :return: self + :return: + self """ self._delete_entries_cache() # Allows to lazily reread on demand. @@ -654,8 +656,9 @@ def _process_diff_args( def _to_relative_path(self, path: PathLike) -> PathLike: """ - :return: Version of path relative to our git directory or raise - :class:`ValueError` if it is not within our git directory. + :return: + Version of path relative to our git directory or raise :class:`ValueError` + if it is not within our git directory. :raise ValueError: """ @@ -693,7 +696,7 @@ def _store_path(self, filepath: PathLike, fprogress: Callable) -> BaseIndexEntry """Store file at filepath in the database and return the base index entry. :note: - This needs the git_working_dir decorator active! + This needs the :func:`~git.index.util.git_working_dir` decorator active! This must be ensured in the calling code. """ st = os.lstat(filepath) # Handles non-symlinks as well. @@ -810,24 +813,25 @@ def add( The handling now very much equals the way string paths are processed, except that the mode you have set will be kept. This allows you to create symlinks by settings the mode respectively and writing the target - of the symlink directly into the file. This equals a default - Linux symlink which is not dereferenced automatically, except that it - can be created on filesystems not supporting it as well. + of the symlink directly into the file. This equals a default Linux + symlink which is not dereferenced automatically, except that it can be + created on filesystems not supporting it as well. Please note that globs or directories are not allowed in - :class:~`git.objects.blob.Blob` objects. + :class:`~git.objects.blob.Blob` objects. They are added at stage 0. - :class:`~git.index.typ.BaseIndexEntry` or type - Handling equals the one of Blob objects, but the stage may be explicitly - set. Please note that Index Entries require binary sha's. + Handling equals the one of :class:~`git.objects.blob.Blob` objects, but + the stage may be explicitly set. Please note that Index Entries require + binary sha's. :param force: **CURRENTLY INEFFECTIVE** - If True, otherwise ignored or excluded files will be added anyway. - As opposed to the ``git add`` command, we enable this flag by default as the + If ``True``, otherwise ignored or excluded files will be added anyway. As + opposed to the ``git add`` command, we enable this flag by default as the API user usually wants the item to be added even though they might be excluded. @@ -835,8 +839,10 @@ def add( Function with signature ``f(path, done=False, item=item)`` called for each path to be added, one time once it is about to be added where ``done=False`` and once after it was added where ``done=True``. - ``item`` is set to the actual item we handle, either a Path or a + + ``item`` is set to the actual item we handle, either a path or a :class:`~git.index.typ.BaseIndexEntry`. + Please note that the processed path is not guaranteed to be present in the index already as the index is currently being processed. @@ -845,24 +851,24 @@ def add( for each passed entry which is the path to be actually recorded for the object created from :attr:`entry.path `. This allows you to write an index which is not identical to the layout of - the actual files on your hard-disk. If not None and `items` contain plain - paths, these paths will be converted to Entries beforehand and passed to the - path_rewriter. Please note that ``entry.path`` is relative to the git + the actual files on your hard-disk. If not ``None`` and `items` contain + plain paths, these paths will be converted to Entries beforehand and passed + to the path_rewriter. Please note that ``entry.path`` is relative to the git repository. :param write: - If True, the index will be written once it was altered. Otherwise the + If ``True``, the index will be written once it was altered. Otherwise the changes only exist in memory and are not available to git commands. :param write_extension_data: - If True, extension data will be written back to the index. This can lead to - issues in case it is containing the 'TREE' extension, which will cause the - ``git commit`` command to write an old tree, instead of a new one + If ``True``, extension data will be written back to the index. This can lead + to issues in case it is containing the 'TREE' extension, which will cause + the ``git commit`` command to write an old tree, instead of a new one representing the now changed index. This doesn't matter if you use :meth:`IndexFile.commit`, which ignores the - 'TREE' extension altogether. You should set it to True if you intend to use - :meth:`IndexFile.commit` exclusively while maintaining support for + 'TREE' extension altogether. You should set it to ``True`` if you intend to + use :meth:`IndexFile.commit` exclusively while maintaining support for third-party extensions. Besides that, you can usually safely ignore the built-in extensions when using GitPython on repositories that are not handled manually at all. @@ -875,7 +881,7 @@ def add( just actually added. :raise OSError: - If a supplied Path did not exist. Please note that + If a supplied path did not exist. Please note that :class:`~git.index.typ.BaseIndexEntry` objects that do not have a null sha will be added even if their paths do not exist. """ @@ -999,7 +1005,7 @@ def remove( it. If absolute paths are given, they will be converted to a path relative to the git repository directory containing the working tree - The path string may include globs, such as \*.c. + The path string may include globs, such as ``*.c``. - :class:~`git.objects.blob.Blob` object @@ -1010,9 +1016,9 @@ def remove( The only relevant information here is the path. The stage is ignored. :param working_tree: - If True, the entry will also be removed from the working tree, physically - removing the respective file. This may fail if there are uncommitted changes - in it. + If ``True``, the entry will also be removed from the working tree, + physically removing the respective file. This may fail if there are + uncommitted changes in it. :param kwargs: Additional keyword arguments to be passed to ``git rm``, such as ``r`` to @@ -1061,7 +1067,7 @@ def move( for reference. :param skip_errors: - If True, errors such as ones resulting from missing source files will be + If ``True``, errors such as ones resulting from missing source files will be skipped. :param kwargs: @@ -1214,21 +1220,21 @@ def checkout( case you have altered the entries dictionary directly. :param paths: - If None, all paths in the index will be checked out. Otherwise an iterable - of relative or absolute paths or a single path pointing to files or - directories in the index is expected. + If ``None``, all paths in the index will be checked out. + Otherwise an iterable of relative or absolute paths or a single path + pointing to files or directories in the index is expected. :param force: - If True, existing files will be overwritten even if they contain local + If ``True``, existing files will be overwritten even if they contain local modifications. - If False, these will trigger a :class:`~git.exc.CheckoutError`. + If ``False``, these will trigger a :class:`~git.exc.CheckoutError`. :param fprogress: See :meth:`IndexFile.add` for signature and explanation. - The provided progress information will contain None as path and item if no - explicit paths are given. Otherwise progress information will be send prior - and after a file has been checked out. + The provided progress information will contain ``None`` as path and item if + no explicit paths are given. Otherwise progress information will be send + prior and after a file has been checked out. :param kwargs: Additional arguments to be passed to ``git checkout-index``. @@ -1238,10 +1244,10 @@ def checkout( guaranteed to match the version stored in the index. :raise git.exc.CheckoutError: - If at least one file failed to be checked out. This is a summary, hence it - will checkout as many files as it can anyway. - If one of files or directories do not exist in the index (as opposed to the - original git command, which ignores them). + * If at least one file failed to be checked out. This is a summary, hence it + will checkout as many files as it can anyway. + * If one of files or directories do not exist in the index (as opposed to + the original git command, which ignores them). :raise git.exc.GitCommandError: If error lines could not be parsed - this truly is an exceptional state. @@ -1394,7 +1400,7 @@ def reset( **kwargs: Any, ) -> "IndexFile": """Reset the index to reflect the tree at the given commit. This will not adjust - our ``HEAD`` reference by default, as opposed to + our HEAD reference by default, as opposed to :meth:`HEAD.reset `. :param commit: @@ -1406,14 +1412,14 @@ def reset( overwrite the default index. :param working_tree: - If True, the files in the working tree will reflect the changed index. - If False, the working tree will not be touched. + If ``True``, the files in the working tree will reflect the changed index. + If ``False``, the working tree will not be touched. Please note that changes to the working copy will be discarded without warning! :param head: - If True, the head will be set to the given commit. This is False by default, - but if True, this method behaves like + If ``True``, the head will be set to the given commit. This is ``False`` by + default, but if ``True``, this method behaves like :meth:`HEAD.reset `. :param paths: @@ -1433,7 +1439,8 @@ def reset( If you want ``git reset``-like behaviour, use :meth:`HEAD.reset ` instead. - :return: self + :return: + self """ # What we actually want to do is to merge the tree into our existing index, # which is what git-read-tree does. diff --git a/git/index/fun.py b/git/index/fun.py index 1d4ca9b93..58335739e 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -222,8 +222,8 @@ def read_header(stream: IO[bytes]) -> Tuple[int, int]: def entry_key(*entry: Union[BaseIndexEntry, PathLike, int]) -> Tuple[PathLike, int]: """ :return: - Key suitable to be used for the :attr:`index.entries - ` dictionary. + Key suitable to be used for the + :attr:`index.entries ` dictionary. :param entry: One instance of type BaseIndexEntry or the path and the stage. diff --git a/git/index/typ.py b/git/index/typ.py index dce67626f..a7d2ad47a 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -36,9 +36,9 @@ class BlobFilter: - """ - Predicate to be used by iter_blobs allowing to filter only return blobs which - match the given list of directories or files. + """Predicate to be used by + :meth:`IndexFile.iter_blobs ` allowing to + filter only return blobs which match the given list of directories or files. The given paths are given relative to the repository. """ @@ -48,8 +48,8 @@ class BlobFilter: def __init__(self, paths: Sequence[PathLike]) -> None: """ :param paths: - Tuple or list of paths which are either pointing to directories or - to files relative to the current repository + Tuple or list of paths which are either pointing to directories or to files + relative to the current repository. """ self.paths = paths @@ -70,7 +70,7 @@ def __call__(self, stage_blob: Tuple[StageType, Blob]) -> bool: class BaseIndexEntryHelper(NamedTuple): - """Typed named tuple to provide named attribute access for BaseIndexEntry. + """Typed named tuple to provide named attribute access for :class:`BaseIndexEntry`. This is needed to allow overriding ``__new__`` in child class to preserve backwards compatibility. @@ -90,12 +90,12 @@ class BaseIndexEntryHelper(NamedTuple): class BaseIndexEntry(BaseIndexEntryHelper): - """Small brother of an index entry which can be created to describe changes + R"""Small brother of an index entry which can be created to describe changes done to the index in which case plenty of additional information is not required. - As the first 4 data members match exactly to the IndexEntry type, methods - expecting a BaseIndexEntry can also handle full IndexEntries even if they - use numeric indices for performance reasons. + As the first 4 data members match exactly to the :class:`IndexEntry` type, methods + expecting a :class:`BaseIndexEntry` can also handle full :class:`IndexEntry`\s even + if they use numeric indices for performance reasons. """ def __new__( @@ -127,9 +127,11 @@ def stage(self) -> int: * 0 = default stage * 1 = stage before a merge or common ancestor entry in case of a 3 way merge * 2 = stage of entries from the 'left' side of the merge - * 3 = stage of entries from the right side of the merge + * 3 = stage of entries from the 'right' side of the merge - :note: For more information, see http://www.kernel.org/pub/software/scm/git/docs/git-read-tree.html + :note: + For more information, see: + http://www.kernel.org/pub/software/scm/git/docs/git-read-tree.html """ return (self.flags & CE_STAGEMASK) >> CE_STAGESHIFT @@ -144,7 +146,8 @@ def to_blob(self, repo: "Repo") -> Blob: class IndexEntry(BaseIndexEntry): - """Allows convenient access to IndexEntry data without completely unpacking it. + """Allows convenient access to index entry data as defined in + :class:`BaseIndexEntry` without completely unpacking it. Attributes usually accessed often are cached in the tuple whereas others are unpacked on demand. @@ -163,17 +166,18 @@ def ctime(self) -> Tuple[int, int]: @property def mtime(self) -> Tuple[int, int]: - """See ctime property, but returns modification time.""" + """See :attr:`ctime` property, but returns modification time.""" return cast(Tuple[int, int], unpack(">LL", self.mtime_bytes)) @classmethod def from_base(cls, base: "BaseIndexEntry") -> "IndexEntry": """ :return: - Minimal entry as created from the given BaseIndexEntry instance. + Minimal entry as created from the given :class:`BaseIndexEntry` instance. Missing values will be set to null-like values. - :param base: Instance of type :class:`BaseIndexEntry` + :param base: + Instance of type :class:`BaseIndexEntry`. """ time = pack(">LL", 0, 0) return IndexEntry((base.mode, base.binsha, base.flags, base.path, time, time, 0, 0, 0, 0, 0)) From 08a80aa1d91af4745dd369db7ad48a23df7145a9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 17:49:11 -0500 Subject: [PATCH 0702/1392] Further revise docstrings within git.objects.submodule --- git/objects/submodule/base.py | 128 +++++++++++++++++++--------------- git/objects/submodule/root.py | 28 ++++---- git/objects/submodule/util.py | 2 +- 3 files changed, 86 insertions(+), 72 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 159403f24..dc1deee36 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -266,8 +266,8 @@ def _clear_cache(self) -> None: def _sio_modules(cls, parent_commit: Commit_ish) -> BytesIO: """ :return: - Configuration file as BytesIO - we only access it through the respective - blob's data + Configuration file as :class:`~io.BytesIO` - we only access it through the + respective blob's data """ sio = BytesIO(parent_commit.tree[cls.k_modules_file].data_stream.read()) sio.name = cls.k_modules_file @@ -354,7 +354,7 @@ def _to_relative_path(cls, parent_repo: "Repo", path: PathLike) -> PathLike: """:return: a path guaranteed to be relative to the given parent repository :raise ValueError: - If path is not contained in the parent repository's working tree + If path is not contained in the parent repository's working tree. """ path = to_native_path_linux(path) if path.endswith("/"): @@ -378,8 +378,8 @@ def _to_relative_path(cls, parent_repo: "Repo", path: PathLike) -> PathLike: @classmethod def _write_git_file_and_module_config(cls, working_tree_dir: PathLike, module_abspath: PathLike) -> None: - """Write a .git file containing a (preferably) relative path to the actual git - module repository. + """Write a ``.git`` file containing a (preferably) relative path to the actual + git module repository. It is an error if the `module_abspath` cannot be made into a relative path, relative to the `working_tree_dir`. @@ -430,10 +430,10 @@ def add( allow_unsafe_options: bool = False, allow_unsafe_protocols: bool = False, ) -> "Submodule": - """Add a new submodule to the given repository. This will alter the index - as well as the ``.gitmodules`` file, but will not create a new commit. - If the submodule already exists, no matter if the configuration differs - from the one provided, the existing submodule will be returned. + """Add a new submodule to the given repository. This will alter the index as + well as the ``.gitmodules`` file, but will not create a new commit. If the + submodule already exists, no matter if the configuration differs from the one + provided, the existing submodule will be returned. :param repo: Repository instance which should receive the submodule. @@ -448,23 +448,23 @@ def add( :param url: git-clone compatible URL, see git-clone reference for more information. - If None, the repository is assumed to exist, and the url of the first remote - is taken instead. This is useful if you want to make an existing repository - a submodule of another one. + If ``None```, the repository is assumed to exist, and the url of the first + remote is taken instead. This is useful if you want to make an existing + repository a submodule of another one. :param branch: Name of branch at which the submodule should (later) be checked out. The given branch must exist in the remote repository, and will be checked out locally as a tracking branch. - It will only be written into the configuration if it not None, which is when - the checked out branch will be the one the remote HEAD pointed to. + It will only be written into the configuration if it not ``None``, which is + when the checked out branch will be the one the remote HEAD pointed to. The result you get in these situation is somewhat fuzzy, and it is recommended to specify at least ``master`` here. Examples are ``master`` or ``feature/new``. :param no_checkout: - If True, and if the repository has to be cloned manually, no checkout will - be performed. + If ``True``, and if the repository has to be cloned manually, no checkout + will be performed. :param depth: Create a shallow clone with a history truncated to the specified number of @@ -479,7 +479,8 @@ def add( want to unset some variable, consider providing an empty string as its value. - :param clone_multi_options: A list of Clone options. Please see + :param clone_multi_options: + A list of clone options. Please see :meth:`Repo.clone ` for details. :param allow_unsafe_protocols: @@ -632,44 +633,49 @@ def update( with the binsha of this instance. :param recursive: - If True, we will operate recursively and update child modules as well. + If ``True``, we will operate recursively and update child modules as well. :param init: - If True, the module repository will be cloned into place if necessary. + If ``True``, the module repository will be cloned into place if necessary. :param to_latest_revision: - If True, the submodule's sha will be ignored during checkout. Instead, the - remote will be fetched, and the local tracking branch updated. This only + If ``True``, the submodule's sha will be ignored during checkout. Instead, + the remote will be fetched, and the local tracking branch updated. This only works if we have a local tracking branch, which is the case if the remote - repository had a master branch, or of the 'branch' option was specified for - this submodule and the branch existed remotely. + repository had a master branch, or of the ``branch`` option was specified + for this submodule and the branch existed remotely. :param progress: - :class:`UpdateProgress` instance, or None if no progress should be shown. + :class:`UpdateProgress` instance, or ``None`` if no progress should be + shown. :param dry_run: - If True, the operation will only be simulated, but not performed. + If ``True``, the operation will only be simulated, but not performed. All performed operations are read-only. :param force: - If True, we may reset heads even if the repository in question is dirty. + If ``True``, we may reset heads even if the repository in question is dirty. Additionally we will be allowed to set a tracking branch which is ahead of its remote branch back into the past or the location of the remote branch. This will essentially 'forget' commits. - If False, local tracking branches that are in the future of their respective - remote branches will simply not be moved. + + If ``False``, local tracking branches that are in the future of their + respective remote branches will simply not be moved. :param keep_going: - If True, we will ignore but log all errors, and keep going recursively. + If ``True``, we will ignore but log all errors, and keep going recursively. Unless `dry_run` is set as well, `keep_going` could cause subsequent/inherited errors you wouldn't see otherwise. In conjunction with `dry_run`, it can be useful to anticipate all errors when updating submodules. - :param env: Optional dictionary containing the desired environment variables. + :param env: + Optional dictionary containing the desired environment variables. + Note: Provided variables will be used to update the execution environment for ``git``. If some variable is not specified in `env` and is defined in attr:`os.environ`, value from attr:`os.environ` will be used. + If you want to unset some variable, consider providing the empty string as its value. @@ -688,7 +694,7 @@ def update( Does nothing in bare repositories. :note: - This method is definitely not atomic if `recursive` is True. + This method is definitely not atomic if `recursive` is ``True``. :return: self @@ -950,18 +956,18 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = :param module_path: The path to which to move our module in the parent repository's working - tree, given as repository - relative or absolute path. Intermediate + tree, given as repository-relative or absolute path. Intermediate directories will be created accordingly. If the path already exists, it must be empty. Trailing (back)slashes are removed automatically. :param configuration: - If True, the configuration will be adjusted to let the submodule point to - the given path. + If ``True``, the configuration will be adjusted to let the submodule point + to the given path. :param module: - If True, the repository managed by this submodule will be moved as well. If - False, we don't move the submodule's checkout, which may leave the parent - repository in an inconsistent state. + If ``True``, the repository managed by this submodule will be moved as well. + If ``False``, we don't move the submodule's checkout, which may leave the + parent repository in an inconsistent state. :return: self @@ -1074,12 +1080,13 @@ def remove( from the ``.gitmodules`` file and the entry in the ``.git/config`` file. :param module: - If True, the checked out module we point to will be deleted as well. If that - module is currently on a commit outside any branch in the remote, or if it - is ahead of its tracking branch, or if there are modified or untracked files - in its working tree, then the removal will fail. In case the removal of the - repository fails for these reasons, the submodule status will not have been - altered. + If ``True``, the checked out module we point to will be deleted as well. If + that module is currently on a commit outside any branch in the remote, or if + it is ahead of its tracking branch, or if there are modified or untracked + files in its working tree, then the removal will fail. In case the removal + of the repository fails for these reasons, the submodule status will not + have been altered. + If this submodule has child modules of its own, these will be deleted prior to touching the direct submodule. @@ -1088,12 +1095,12 @@ def remove( This basically enforces a brute-force file system based deletion. :param configuration: - If True, the submodule is deleted from the configuration, otherwise it + If ``True``, the submodule is deleted from the configuration, otherwise it isn't. Although this should be enabled most of the time, this flag enables you to safely delete the repository of your submodule. :param dry_run: - If True, we will not actually do anything, but throw the errors we would + If ``True``, we will not actually do anything, but throw the errors we would usually throw. :return: @@ -1239,12 +1246,12 @@ def set_parent_commit(self, commit: Union[Commit_ish, None], check: bool = True) contain the ``.gitmodules`` blob. :param commit: - Commit-ish reference pointing at the root_tree, or None to always point to - the most recent commit + Commit-ish reference pointing at the root_tree, or ``None`` to always point + to the most recent commit. :param check: - If True, relatively expensive checks will be performed to verify validity of - the submodule. + If ``True``, relatively expensive checks will be performed to verify + validity of the submodule. :raise ValueError: If the commit's tree didn't contain the ``.gitmodules`` blob. @@ -1297,8 +1304,9 @@ def config_writer( to this submodule into the ``.gitmodules`` file. :param index: - If not None, an IndexFile instance which should be written. - Defaults to the index of the Submodule's parent repository. + If not ``None``, an :class:`~git.index.base.IndexFile` instance which should + be written. Defaults to the index of the :class:`Submodule`'s parent + repository. :param write: If True, the index will be written each time a configuration value changes. @@ -1380,7 +1388,9 @@ def rename(self, new_name: str) -> "Submodule": @unbare_repo def module(self) -> "Repo": """ - :return: Repo instance initialized from the repository at our submodule path + :return: + :class:`~git.repo.base.Repo` instance initialized from the repository at our + submodule path :raise git.exc.InvalidGitRepositoryError: If a repository was not available. @@ -1401,7 +1411,7 @@ def module(self) -> "Repo": def module_exists(self) -> bool: """ :return: - True if our module exists and is a valid git repository. + ``True`` if our module exists and is a valid git repository. See the :meth:`module` method. """ try: @@ -1414,7 +1424,7 @@ def module_exists(self) -> bool: def exists(self) -> bool: """ :return: - True if the submodule exists, False otherwise. + ``True`` if the submodule exists, ``False`` otherwise. Please note that a submodule may exist (in the ``.gitmodules`` file) even though its module doesn't exist on disk. """ @@ -1480,14 +1490,15 @@ def branch_name(self) -> str: @property def url(self) -> str: - """:return: The url to the repository which our module - repository refers to""" + """:return: The url to the repository our submodule's repository refers to""" return self._url @property def parent_commit(self) -> "Commit_ish": """ :return: - Commit instance with the tree containing the ``.gitmodules`` file + :class:`~git.objects.commit.Commit` instance with the tree containing the + ``.gitmodules`` file :note: Will always point to the current head's commit if it was not set explicitly. @@ -1531,8 +1542,9 @@ def config_reader(self) -> SectionConstraint[SubmoduleConfigParser]: def children(self) -> IterableList["Submodule"]: """ :return: - IterableList(Submodule, ...) An iterable list of submodules instances which - are children of this submodule or 0 if the submodule is not checked out. + IterableList(Submodule, ...) An iterable list of :class:`Submodule` + instances which are children of this submodule or 0 if the submodule is not + checked out. """ return self._get_intermediate_items(self) diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index 8ef874f90..ac0f7ad94 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -89,9 +89,9 @@ def update( ) -> "RootModule": """Update the submodules of this repository to the current HEAD commit. - This method behaves smartly by determining changes of the path of a submodules + This method behaves smartly by determining changes of the path of a submodule's repository, next to changes to the to-be-checked-out commit or the branch to be - checked out. This works if the submodules ID does not change. + checked out. This works if the submodule's ID does not change. Additionally it will detect addition and removal of submodules, which will be handled gracefully. @@ -99,11 +99,11 @@ def update( :param previous_commit: If set to a commit-ish, the commit we should use as the previous commit the HEAD pointed to before it was set to the commit it points to now. - If None, it defaults to ``HEAD@{1}`` otherwise. + If ``None``, it defaults to ``HEAD@{1}`` otherwise. :param recursive: - If True, the children of submodules will be updated as well using the same - technique. + If ``True``, the children of submodules will be updated as well using the + same technique. :param force_remove: If submodules have been deleted, they will be forcibly removed. Otherwise @@ -116,28 +116,30 @@ def update( If we encounter a new module which would need to be initialized, then do it. :param to_latest_revision: - If True, instead of checking out the revision pointed to by this submodule's - sha, the checked out tracking branch will be merged with the latest remote - branch fetched from the repository's origin. + If ``True``, instead of checking out the revision pointed to by this + submodule's sha, the checked out tracking branch will be merged with the + latest remote branch fetched from the repository's origin. + Unless `force_reset` is specified, a local tracking branch will never be reset into its past, therefore the remote branch must be in the future for this to have an effect. :param force_reset: - If True, submodules may checkout or reset their branch even if the + If ``True``, submodules may checkout or reset their branch even if the repository has pending changes that would be overwritten, or if the local tracking branch is in the future of the remote tracking branch and would be reset into its past. :param progress: - :class:`RootUpdateProgress` instance or None if no progress should be sent. + :class:`RootUpdateProgress` instance, or ``None`` if no progress should be + sent. :param dry_run: - If True, operations will not actually be performed. Progress messages will - change accordingly to indicate the WOULD DO state of the operation. + If ``True``, operations will not actually be performed. Progress messages + will change accordingly to indicate the WOULD DO state of the operation. :param keep_going: - If True, we will ignore but log all errors, and keep going recursively. + If ``True``, we will ignore but log all errors, and keep going recursively. Unless `dry_run` is set as well, `keep_going` could cause subsequent/inherited errors you wouldn't see otherwise. In conjunction with `dry_run`, this can be useful to anticipate all errors diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index b02da501e..e5af5eb31 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -52,7 +52,7 @@ def mkhead(repo: "Repo", path: PathLike) -> "Head": def find_first_remote_branch(remotes: Sequence["Remote"], branch_name: str) -> "RemoteReference": """Find the remote branch matching the name of the given branch or raise - :class:`git.exc.InvalidGitRepositoryError`.""" + :class:`~git.exc.InvalidGitRepositoryError`.""" for remote in remotes: try: return remote.refs[branch_name] From bc48d264df657f3d5593760365427467980b1ef9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 18:46:22 -0500 Subject: [PATCH 0703/1392] Further revise other docstrings within git.objects --- git/objects/base.py | 12 ++++++------ git/objects/blob.py | 3 ++- git/objects/commit.py | 28 ++++++++++++++-------------- git/objects/fun.py | 14 ++++++++------ git/objects/tree.py | 23 +++++++++++------------ git/objects/util.py | 37 +++++++++++++++++++------------------ 6 files changed, 60 insertions(+), 57 deletions(-) diff --git a/git/objects/base.py b/git/objects/base.py index df56a4bac..5cee8e405 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -58,7 +58,7 @@ class Object(LazyMixin): def __init__(self, repo: "Repo", binsha: bytes): """Initialize an object by identifying it by its binary sha. - All keyword arguments will be set on demand if None. + All keyword arguments will be set on demand if ``None``. :param repo: Repository this object is located in. @@ -83,7 +83,7 @@ def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> Commit_ish: input id may have been a `~git.refs.reference.Reference` or rev-spec. :param id: - :class:`~git.refs.reference.Reference`, rev-spec, or hexsha + :class:`~git.refs.reference.Reference`, rev-spec, or hexsha. :note: This cannot be a ``__new__`` method as it would always call :meth:`__init__` @@ -119,13 +119,13 @@ def _set_cache_(self, attr: str) -> None: super()._set_cache_(attr) def __eq__(self, other: Any) -> bool: - """:return: True if the objects have the same SHA1""" + """:return: ``True`` if the objects have the same SHA1""" if not hasattr(other, "binsha"): return False return self.binsha == other.binsha def __ne__(self, other: Any) -> bool: - """:return: True if the objects do not have the same SHA1""" + """:return: ``True`` if the objects do not have the same SHA1""" if not hasattr(other, "binsha"): return True return self.binsha != other.binsha @@ -199,8 +199,8 @@ def __init__( 20 byte sha1. :param mode: - The stat compatible file mode as int, use the :mod:`stat` module to evaluate - the information. + The stat-compatible file mode as :class:`int`. + Use the :mod:`stat` module to evaluate the information. :param path: The path to the file in the file system, relative to the git repository diff --git a/git/objects/blob.py b/git/objects/blob.py index 50c12f0d7..253ceccb5 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -27,7 +27,8 @@ class Blob(base.IndexObject): @property def mime_type(self) -> str: """ - :return: String describing the mime type of this file (based on the filename) + :return: + String describing the mime type of this file (based on the filename) :note: Defaults to ``text/plain`` in case the actual file type is unknown. diff --git a/git/objects/commit.py b/git/objects/commit.py index e6aa7ac39..b5b1c540d 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -108,11 +108,11 @@ def __init__( encoding: Union[str, None] = None, gpgsig: Union[str, None] = None, ) -> None: - R"""Instantiate a new :class:`Commit`. All keyword arguments taking None as + R"""Instantiate a new :class:`Commit`. All keyword arguments taking ``None`` as default will be implicitly set on first query. :param binsha: - 20 byte sha1 + 20 byte sha1. :param parents: tuple(Commit, ...) A tuple of commit ids or actual :class:`Commit`\s. @@ -120,8 +120,8 @@ def __init__( :param tree: A :class:`~git.objects.tree.Tree` object. - :param author: :class:`~git.util.Actor` - The author Actor object + :param author: + The author :class:`~git.util.Actor` object. :param authored_date: int_seconds_since_epoch The authored DateTime - use :func:`time.gmtime` to convert it into a @@ -130,8 +130,8 @@ def __init__( :param author_tz_offset: int_seconds_west_of_utc The timezone that the `authored_date` is in. - :param committer: :class:`~git.util.Actor` - The committer string. + :param committer: + The committer string, as an :class:`~git.util.Actor` object. :param committed_date: int_seconds_since_epoch The committed DateTime - use :func:`time.gmtime` to convert it into a @@ -209,7 +209,7 @@ def _calculate_sha_(cls, repo: "Repo", commit: "Commit") -> bytes: return istream.binsha def replace(self, **kwargs: Any) -> "Commit": - """Create new commit object from existing commit object. + """Create new commit object from an existing commit object. Any values provided as keyword arguments will replace the corresponding attribute in the new object. @@ -295,7 +295,7 @@ def iter_items( R"""Find all commits matching the given criteria. :param repo: - The Repo + The :class:`~git.repo.base.Repo`. :param rev: Revision specifier, see git-rev-parse for viable options. @@ -378,7 +378,7 @@ def stats(self) -> Stats: @property def trailers(self) -> Dict[str, str]: - """Get the trailers of the message as a dictionary + """Deprecated. Get the trailers of the message as a dictionary. :note: This property is deprecated, please use either :attr:`trailers_list` or :attr:`trailers_dict``. @@ -396,7 +396,7 @@ def trailers_list(self) -> List[Tuple[str, str]]: Git messages can contain trailer information that are similar to RFC 822 e-mail headers (see: https://git-scm.com/docs/git-interpret-trailers). - This functions calls ``git interpret-trailers --parse`` onto the message to + This function calls ``git interpret-trailers --parse`` onto the message to extract the trailer information, returns the raw trailer data as a list. Valid message with trailer:: @@ -444,7 +444,7 @@ def trailers_dict(self) -> Dict[str, List[str]]: Git messages can contain trailer information that are similar to RFC 822 e-mail headers (see: https://git-scm.com/docs/git-interpret-trailers). - This functions calls ``git interpret-trailers --parse`` onto the message to + This function calls ``git interpret-trailers --parse`` onto the message to extract the trailer information. The key value pairs are stripped of leading and trailing whitespaces before they get saved into a dictionary. @@ -481,7 +481,7 @@ def trailers_dict(self) -> Dict[str, List[str]]: def _iter_from_process_or_stream(cls, repo: "Repo", proc_or_stream: Union[Popen, IO]) -> Iterator["Commit"]: """Parse out commit information into a list of :class:`Commit` objects. - We expect one-line per commit, and parse the actual commit information directly + We expect one line per commit, and parse the actual commit information directly from our lighting fast object database. :param proc: @@ -555,11 +555,11 @@ def create_from_tree( :param parent_commits: Optional :class:`Commit` objects to use as parents for the new commit. If empty list, the commit will have no parents at all and become a root commit. - If None, the current head commit will be the parent of the new commit + If ``None``, the current head commit will be the parent of the new commit object. :param head: - If True, the HEAD will be advanced to the new commit automatically. + If ``True``, the HEAD will be advanced to the new commit automatically. Otherwise the HEAD will remain pointing on the previous commit. This could lead to undesired results when diffing files. diff --git a/git/objects/fun.py b/git/objects/fun.py index d349737de..ad5bc9a88 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -128,7 +128,7 @@ def tree_entries_from_data(data: bytes) -> List[EntryTup]: def _find_by_name(tree_data: MutableSequence[EntryTupOrNone], name: str, is_dir: bool, start_at: int) -> EntryTupOrNone: - """Return data entry matching the given name and tree mode or None. + """Return data entry matching the given name and tree mode or ``None``. Before the item is returned, the respective data item is set None in the `tree_data` list to mark it done. @@ -172,7 +172,8 @@ def traverse_trees_recursive( odb: "GitCmdObjectDB", tree_shas: Sequence[Union[bytes, None]], path_prefix: str ) -> List[Tuple[EntryTupOrNone, ...]]: """ - :return: list of list with entries according to the given binary tree-shas. + :return: + List of list with entries according to the given binary tree-shas. The result is encoded in a list of n tuple|None per blob/commit, (n == len(tree_shas)), where: @@ -181,12 +182,12 @@ def traverse_trees_recursive( * [1] == mode as int * [2] == path relative to working tree root - The entry tuple is None if the respective blob/commit did not exist in the given - tree. + The entry tuple is ``None`` if the respective blob/commit did not exist in the + given tree. :param tree_shas: Iterable of shas pointing to trees. All trees must be on the same level. A - tree-sha may be None in which case None. + tree-sha may be ``None`` in which case ``None``. :param path_prefix: A prefix to be added to the returned paths on this level. @@ -257,7 +258,8 @@ def traverse_trees_recursive( def traverse_tree_recursive(odb: "GitCmdObjectDB", tree_sha: bytes, path_prefix: str) -> List[EntryTup]: """ - :return: list of entries of the tree pointed to by the binary `tree_sha`. + :return: + List of entries of the tree pointed to by the binary `tree_sha`. An entry has the following format: diff --git a/git/objects/tree.py b/git/objects/tree.py index da682b8cd..69cd6ef3f 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -97,17 +97,17 @@ def add(self, sha: bytes, mode: int, name: str, force: bool = False) -> "TreeMod If an item with the given name already exists, nothing will be done, but a :class:`ValueError` will be raised if the sha and mode of the existing item do - not match the one you add, unless `force` is True. + not match the one you add, unless `force` is ``True``. :param sha: The 20 or 40 byte sha of the item to add. :param mode: - int representing the stat compatible mode of the item. + :class:`int` representing the stat-compatible mode of the item. :param force: - If True, an item with your name and information will overwrite any existing - item with the same name, no matter which information it has. + If ``True``, an item with your name and information will overwrite any + existing item with the same name, no matter which information it has. :return: self @@ -164,11 +164,10 @@ class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): R"""Tree objects represent an ordered list of :class:`~git.objects.blob.Blob`\s and other :class:`~git.objects.tree.Tree`\s. - Tree as a list:: + Tree as a list: - Access a specific blob using the ``tree['filename']`` notation. - - You may likewise access by index, like ``blob = tree[0]``. + * Access a specific blob using the ``tree['filename']`` notation. + * You may likewise access by index, like ``blob = tree[0]``. """ type: Literal["tree"] = "tree" @@ -235,7 +234,7 @@ def join(self, file: str) -> IndexObjUnion: or :class:`~git.objects.submodule.base.Submodule` :raise KeyError: - If the given file or tree does not exist in tree. + If the given file or tree does not exist in this tree. """ msg = "Blob or Tree named %r not found" if "/" in file: @@ -275,12 +274,12 @@ def __truediv__(self, file: str) -> IndexObjUnion: @property def trees(self) -> List["Tree"]: - """:return: list(Tree, ...) list of trees directly below this tree""" + """:return: list(Tree, ...) List of trees directly below this tree""" return [i for i in self if i.type == "tree"] @property def blobs(self) -> List[Blob]: - """:return: list(Blob, ...) list of blobs directly below this tree""" + """:return: list(Blob, ...) List of blobs directly below this tree""" return [i for i in self if i.type == "blob"] @property @@ -342,7 +341,7 @@ def list_traverse(self, *args: Any, **kwargs: Any) -> IterableList[IndexObjUnion :class:`~git.util.IterableList`IterableList with the results of the traversal as produced by :meth:`traverse` - Tree -> IterableList[Union['Submodule', 'Tree', 'Blob']] + Tree -> IterableList[Union[Submodule, Tree, Blob]] """ return super()._list_traverse(*args, **kwargs) diff --git a/git/objects/util.py b/git/objects/util.py index 7ad20d66e..6f46bab18 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -95,9 +95,9 @@ def mode_str_to_int(modestr: Union[bytes, str]) -> int: used. :return: - String identifying a mode compatible to the mode methods ids of the stat module - regarding the rwx permissions for user, group and other, special flags and file - system flags, such as whether it is a symlink. + String identifying a mode compatible to the mode methods ids of the :mod:`stat` + module regarding the rwx permissions for user, group and other, special flags + and file system flags, such as whether it is a symlink. """ mode = 0 for iteration, char in enumerate(reversed(modestr[-6:])): @@ -401,7 +401,7 @@ class Tree:: (cls, Tree) -> Tuple[Tree, ...] def list_traverse(self, *args: Any, **kwargs: Any) -> Any: """Traverse self and collect all items found. - Calling this directly only the abstract base class, including via a ``super()`` + Calling this directly on the abstract base class, including via a ``super()`` proxy, is deprecated. Only overridden implementations should be called. """ warnings.warn( @@ -418,12 +418,13 @@ def _list_traverse( ) -> IterableList[Union["Commit", "Submodule", "Tree", "Blob"]]: """Traverse self and collect all items found. - :return: :class:`~git.util.IterableList` with the results of the traversal as + :return: + :class:`~git.util.IterableList` with the results of the traversal as produced by :meth:`traverse`:: - Commit -> IterableList["Commit"] - Submodule -> IterableList["Submodule"] - Tree -> IterableList[Union["Submodule", "Tree", "Blob"]] + Commit -> IterableList[Commit] + Submodule -> IterableList[Submodule] + Tree -> IterableList[Union[Submodule, Tree, Blob]] """ # Commit and Submodule have id.__attribute__ as IterableObj. # Tree has id.__attribute__ inherited from IndexObject. @@ -476,12 +477,12 @@ def _traverse( """Iterator yielding items found when traversing self. :param predicate: - A function ``f(i,d)`` that returns False if item i at depth ``d`` should not - be included in the result. + A function ``f(i,d)`` that returns ``False`` if item i at depth ``d`` should + not be included in the result. :param prune: - A function ``f(i,d)`` that returns True if the search should stop at item - ``i`` at depth ``d``. Item ``i`` will not be returned. + A function ``f(i,d)`` that returns ``True`` if the search should stop at + item ``i`` at depth ``d``. Item ``i`` will not be returned. :param depth: Defines at which level the iteration should not go deeper if -1, there is no @@ -489,19 +490,19 @@ def _traverse( i.e. if 1, you would only get the first level of predecessors/successors. :param branch_first: - If True, items will be returned branch first, otherwise depth first. + If ``True``, items will be returned branch first, otherwise depth first. :param visit_once: - If True, items will only be returned once, although they might be + If ``True``, items will only be returned once, although they might be encountered several times. Loops are prevented that way. :param ignore_self: - If True, self will be ignored and automatically pruned from the result. - Otherwise it will be the first item to be returned. If `as_edge` is True, the - source of the first edge is None + If ``True``, self will be ignored and automatically pruned from the result. + Otherwise it will be the first item to be returned. If `as_edge` is + ``True``, the source of the first edge is ``None``. :param as_edge: - If True, return a pair of items, first being the source, second the + If ``True``, return a pair of items, first being the source, second the destination, i.e. tuple(src, dest) with the edge spanning from source to destination. From 30f7da508154df8b968f171b789c366c52276cee Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 18:50:46 -0500 Subject: [PATCH 0704/1392] Fix erroneous reference to DateTime "class" DateTime is not a class here, and the parameter is expected as int. This fixes a documentation bug I introduced in cd16a35 (#1725). --- git/objects/tag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/tag.py b/git/objects/tag.py index 211c84ac7..357c8ff7e 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -68,7 +68,7 @@ def __init__( :class:`~git.util.Actor` identifying the tagger. :param tagged_date: int_seconds_since_epoch - The :class:`DateTime` of the tag creation. + The DateTime of the tag creation. Use :func:`time.gmtime` to convert it into a different format. :param tagger_tz_offset: int_seconds_west_of_utc From 61269970f2b88ba008e5b8c5e4d5e4d2975d82d7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 19:06:07 -0500 Subject: [PATCH 0705/1392] Improve docstrings about tags - Fix long-outdated information in the git.objects.tag docstring that seems to have been left over from when there was no such separate module. This module has only a tag-related class, not all classes that derive from git.object.base.Object. - Expand the git.objects.tag docstring to clarify what the module provides, add a git.refs.tag module docstring explaining what that provides, and cross-reference them with an explanation of how they differ (mentioning how this relates to git concepts). - Expand thie git.object.tag.TagObject class docstring to include the term "annotated" (retaining "non-lightweight" in parentheses). --- git/objects/tag.py | 10 +++++++--- git/refs/tag.py | 7 +++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/git/objects/tag.py b/git/objects/tag.py index 357c8ff7e..7589a5be1 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -3,7 +3,11 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Object-based types.""" +"""Provides an :class:`git.objects.base.Object`-based type for annotated tags. + +This defines the :class:`TagReference` class, which represents annotated tags. +For lightweight tags, see the :mod:`git.refs.tag` module. +""" from . import base from .util import get_object_type_by_name, parse_actor_and_date @@ -25,8 +29,8 @@ class TagObject(base.Object): - """Non-lightweight tag carrying additional information about an object we are - pointing to.""" + """Annotated (i.e. non-lightweight) tag carrying additional information about an + object we are pointing to.""" type: Literal["tag"] = "tag" diff --git a/git/refs/tag.py b/git/refs/tag.py index 7edd1d12a..99c4bf443 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -1,6 +1,13 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +"""Provides a :class:`~git.refs.reference.Reference`-based type for lightweight tags. + +This defines the :class:`TagReference` class (and its alias :class:`Tag`), which +represents lightweight tags. For annotated tags (which are git objects), see the +:mod:`git.objects.tag` module. +""" + from .reference import Reference __all__ = ["TagReference", "Tag"] From 110706e2f49eb90b29ceb3057340437d3d93245f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 19:25:50 -0500 Subject: [PATCH 0706/1392] Fix param name in TagRefernece docstring and add info The "reference" parameter was listed as "ref" in the docstring. This fixes that, slightly expands its description, and also adds documentation for the "repo" parameter. --- git/refs/tag.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/git/refs/tag.py b/git/refs/tag.py index 99c4bf443..89a82d1be 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -72,7 +72,8 @@ def commit(self) -> "Commit": # type: ignore[override] # LazyMixin has unrelat def tag(self) -> Union["TagObject", None]: """ :return: - Tag object this tag ref points to or None in case we are a lightweight tag + Tag object this tag ref points to, or ``None`` in case we are a lightweight + tag """ obj = self.object if obj.type == "tag": @@ -96,13 +97,17 @@ def create( ) -> "TagReference": """Create a new tag reference. + :param repo: + The :class:`~git.repo.base.Repo` to create the tag in. + :param path: The name of the tag, e.g. ``1.0`` or ``releases/1.0``. The prefix ``refs/tags`` is implied. - :param ref: + :param reference: A reference to the :class:`~git.objects.base.Object` you want to tag. - The Object can be a commit, tree or blob. + The referenced object can be a :class:`~git.objects.commit.Commit`, + :class:`~git.objects.tree.Tree`, or :class:`~git.objects.blob.Blob`. :param logmsg: If not None, the message will be used in your tag object. This will also From b0e5bffef39ae6064bb2a1909bc0b536244b4b4c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 19:29:29 -0500 Subject: [PATCH 0707/1392] Undo some expansion of "reference" parameter This keeps most of the changes from the previous commit, but it undoes the expansion of the git object types a lightweight tag can refer to into GitPython git.objects.base.Object-based types, which seems more misleading than helpful because an uncareful reading could lead to an incorrect belief that the corresponding argument should or could be an object of such types. --- git/refs/tag.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/git/refs/tag.py b/git/refs/tag.py index 89a82d1be..fd2dee131 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -106,8 +106,7 @@ def create( :param reference: A reference to the :class:`~git.objects.base.Object` you want to tag. - The referenced object can be a :class:`~git.objects.commit.Commit`, - :class:`~git.objects.tree.Tree`, or :class:`~git.objects.blob.Blob`. + The referenced object can be a commit, tree, or blob. :param logmsg: If not None, the message will be used in your tag object. This will also From a5a1b2c541072ee73b5c295581f678acb7c91c2c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 19:34:11 -0500 Subject: [PATCH 0708/1392] Add a bit of missing docstring formatting --- git/refs/tag.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/refs/tag.py b/git/refs/tag.py index fd2dee131..6a6dad09a 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -109,7 +109,7 @@ def create( The referenced object can be a commit, tree, or blob. :param logmsg: - If not None, the message will be used in your tag object. This will also + If not ``None``, the message will be used in your tag object. This will also create an additional tag object that allows to obtain that information, e.g.:: @@ -120,7 +120,7 @@ def create( `logmsg` takes precedence if both are passed. :param force: - If True, force creation of a tag even though that tag already exists. + If ``True``, force creation of a tag even though that tag already exists. :param kwargs: Additional keyword arguments to be passed to ``git tag``. From 018ebaf72d2d96b2ca98a46748e857abb829a9d4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 19:53:46 -0500 Subject: [PATCH 0709/1392] Further revise docstrings within git.repo --- git/repo/base.py | 74 ++++++++++++++++++++++++++---------------------- git/repo/fun.py | 8 +++--- 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index 7b3e514c1..4745a2411 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -102,9 +102,9 @@ class BlameEntry(NamedTuple): class Repo: - """Represents a git repository and allows you to query references, - gather commit information, generate diffs, create and clone repositories, and query - the log. + """Represents a git repository and allows you to query references, father + commit information, generate diffs, create and clone repositories, and query the + log. The following attributes are worth using: @@ -188,16 +188,18 @@ def __init__( repo = Repo(R"C:\Users\mtrier\Development\git-python\.git") - In *Cygwin*, `path` may be a ``cygdrive/...`` prefixed path. - - If `path` is None or an empty string, :envvar:`GIT_DIR` is used. If that - environment variable is absent or empty, the current directory is used. + - If `path` is ``None`` or an empty string, :envvar:`GIT_DIR` is used. If + that environment variable is absent or empty, the current directory is + used. :param odbt: Object DataBase type - a type which is constructed by providing the directory containing the database objects, i.e. ``.git/objects``. It will be - used to access all object data + used to access all object data. :param search_parent_directories: - If True, all parent directories will be searched for a valid repo as well. + If ``True``, all parent directories will be searched for a valid repo as + well. Please note that this was the default behaviour in older versions of GitPython, which is considered a bug though. @@ -372,7 +374,7 @@ def working_tree_dir(self) -> Optional[PathLike]: """ :return: The working tree directory of our git repository. - If this is a bare repository, None is returned. + If this is a bare repository, ``None`` is returned. """ return self._working_tree_dir @@ -387,7 +389,7 @@ def common_dir(self) -> PathLike: @property def bare(self) -> bool: - """:return: True if the repository is bare""" + """:return: ``True`` if the repository is bare""" return self._bare @property @@ -450,7 +452,7 @@ def remotes(self) -> "IterableList[Remote]": def remote(self, name: str = "origin") -> "Remote": """:return: The remote with the specified name - :raise ValueError + :raise ValueError: If no remote with such a name exists. """ r = Remote(self, name) @@ -494,10 +496,13 @@ def create_submodule(self, *args: Any, **kwargs: Any) -> Submodule: return Submodule.add(self, *args, **kwargs) def iter_submodules(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: - """An iterator yielding Submodule instances, see Traversable interface - for a description of args and kwargs. + """An iterator yielding Submodule instances. - :return: Iterator + See the `~git.objects.util.Traversable` interface for a description of `args` + and `kwargs`. + + :return: + Iterator """ return RootModule(self).traverse(*args, **kwargs) @@ -554,7 +559,8 @@ def create_head( ) -> "Head": """Create a new head within the repository. - :note: For more documentation, please see the + :note: + For more documentation, please see the :meth:`Head.create ` method. :return: @@ -648,7 +654,7 @@ def config_reader( configuration files. :param config_level: - For possible values, see the :meth:`config_writer` method. If None, all + For possible values, see the :meth:`config_writer` method. If ``None``, all applicable levels will be used. Specify a level in case you know which file you wish to read to prevent reading multiple files. @@ -744,7 +750,7 @@ def iter_commits( :param rev: Revision specifier, see ``git rev-parse`` for viable options. - If None, the active branch will be used. + If ``None``, the active branch will be used. :param paths: An optional path or a list of paths. If set, only commits that include the @@ -759,7 +765,7 @@ def iter_commits( ``"revA...revB"`` revision specifier. :return: - Iterator of :class:`~git.objects.commit.Commit` objects + Iterator of :class:`~git.objects.commit.Commit` objects """ if rev is None: rev = self.head.commit @@ -819,7 +825,7 @@ def is_ancestor(self, ancestor_rev: "Commit", rev: "Commit") -> bool: Rev to test against `ancestor_rev`. :return: - True if `ancestor_rev` is an ancestor to `rev`. + ``True`` if `ancestor_rev` is an ancestor to `rev`. """ try: self.git.merge_base(ancestor_rev, rev, is_ancestor=True) @@ -923,9 +929,9 @@ def is_dirty( ) -> bool: """ :return: - True if the repository is considered dirty. By default it will react like a - git-status without untracked files, hence it is dirty if the index or the - working copy have changes. + ``True`` if the repository is considered dirty. By default it will react + like a git-status without untracked files, hence it is dirty if the index or + the working copy have changes. """ if self._bare: # Bare repositories with no associated working directory are @@ -1036,9 +1042,9 @@ def blame_incremental(self, rev: str | HEAD | None, file: str, **kwargs: Any) -> stream of :class:`BlameEntry` tuples. :param rev: - Revision specifier. If None, the blame will include all the latest - uncommitted changes. Otherwise, anything successfully parsed by ``git - rev-parse`` is a valid option. + Revision specifier. If ``None``, the blame will include all the latest + uncommitted changes. Otherwise, anything successfully parsed by + ``git rev-parse`` is a valid option. :return: Lazy iterator of :class:`BlameEntry` tuples, where the commit indicates the @@ -1132,9 +1138,9 @@ def blame( """The blame information for the given file at the given revision. :param rev: - Revision specifier. If None, the blame will include all the latest + Revision specifier. If ``None``, the blame will include all the latest uncommitted changes. Otherwise, anything successfully parsed by - git-rev-parse is a valid option. + ``git rev-parse`` is a valid option. :return: list: [git.Commit, list: []] @@ -1286,9 +1292,9 @@ def init( """Initialize a git repository at the given path if specified. :param path: - The full path to the repo (traditionally ends with ``/.git``). - Or None, in which case the repository will be created in the current working - directory. + The full path to the repo (traditionally ends with ``/.git``). Or + ``None``, in which case the repository will be created in the current + working directory. :param mkdir: If specified, will create the repository directory if it doesn't already @@ -1445,7 +1451,7 @@ def clone( Allow unsafe options to be used, like ``--upload-pack``. :param kwargs: - * odbt = ObjectDatabase Type, allowing to determine the object database + * ``odbt`` = ObjectDatabase Type, allowing to determine the object database implementation used by the returned Repo instance. * All remaining keyword arguments are given to the ``git clone`` command. @@ -1547,9 +1553,9 @@ def archive( :param kwargs: Additional arguments passed to ``git archive``: - * Use the 'format' argument to define the kind of format. Use specialized + * Use the ``format`` argument to define the kind of format. Use specialized ostreams to write any format supported by Python. - * You may specify the special **path** keyword, which may either be a + * You may specify the special ``path`` keyword, which may either be a repository-relative path to a directory or file to place into the archive, or a list or tuple of multiple paths. @@ -1581,7 +1587,7 @@ def has_separate_working_tree(self) -> bool: to. :note: - Bare repositories will always return False here. + Bare repositories will always return ``False`` here. """ if self.bare: return False @@ -1601,7 +1607,7 @@ def currently_rebasing_on(self) -> Commit | None: :return: The commit which is currently being replayed while rebasing. - None if we are not currently rebasing. + ``None`` if we are not currently rebasing. """ if self.git_dir: rebase_head_file = osp.join(self.git_dir, "REBASE_HEAD") diff --git a/git/repo/fun.py b/git/repo/fun.py index 4936f1773..e9fad2c46 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -128,7 +128,7 @@ def find_submodule_git_dir(d: "PathLike") -> Optional["PathLike"]: def short_to_long(odb: "GitCmdObjectDB", hexsha: str) -> Optional[bytes]: """ :return: - Long hexadecimal sha1 from the given less than 40 byte hexsha or None if no + Long hexadecimal sha1 from the given less than 40 byte hexsha, or ``None`` if no candidate could be found. :param hexsha: @@ -150,7 +150,7 @@ def name_to_object( references are supported. :param return_ref: - If True, and name specifies a reference, we will return the reference + If ``True``, and name specifies a reference, we will return the reference instead of the object. Otherwise it will raise `~gitdb.exc.BadObject` o `~gitdb.exc.BadName`. """ @@ -202,7 +202,7 @@ def name_to_object( def deref_tag(tag: "Tag") -> "TagObject": - """Recursively dereference a tag and return the resulting object""" + """Recursively dereference a tag and return the resulting object.""" while True: try: tag = tag.object @@ -213,7 +213,7 @@ def deref_tag(tag: "Tag") -> "TagObject": def to_commit(obj: Object) -> Union["Commit", "TagObject"]: - """Convert the given object to a commit if possible and return it""" + """Convert the given object to a commit if possible and return it.""" if obj.type == "tag": obj = deref_tag(obj) From 608147ec7fa240686cc88612e1962c18fc6e8549 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 20:07:13 -0500 Subject: [PATCH 0710/1392] Better explain conditional cleanup in test_base_object This expands the comment added in 41fac85 (#1770) to make more clear that this particular cleanup is deliberately done only when the operation was successful (and why). --- test/test_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_base.py b/test/test_base.py index 74f342071..693c97f20 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -72,7 +72,10 @@ def test_base_object(self): self.assertEqual(item, item.stream_data(tmpfile)) tmpfile.seek(0) self.assertEqual(tmpfile.read(), data) - os.remove(tmpfile.name) # Do it this way so we can inspect the file on failure. + + # Remove the file this way, instead of with a context manager or "finally", + # so it is only removed on success, and we can inspect the file on failure. + os.remove(tmpfile.name) # END for each object type to create # Each has a unique sha. From 5cf5b6049f4e4cc83b08c1b2f36aaf0ad48b67f1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 20:48:52 -0500 Subject: [PATCH 0711/1392] Revise test suite docstrings and comments In a more limited way than in the git/ code. Docstring revisions are, in spirit, along the same lines as those in the git package, but much more conservative, because the tests' docstrings are not rendered into Sphinx documentation and there is no current plan to do so (so the tradeoffs are different, with even a tiny decrease in clarity when read in the code being a reason to avoid changes that use Sphinx roles more robustly or that would improve hypothetical rendered documentation), and because the test suite uses docstrings more sparingly and the existing docstrings were mostly already clear and easy to read. This wraps commnts to 88 columns as most comments now are in the git package, except it avoids doing so when doing so would make anything even slightly less clear, and where it would require significant further style or spacing changes for it to remain obvious (even before reading a comment) what the comment applies to, and in most portions of tutorial-generating test case code (where, although clarity would improve when reading the tests, it might sometimes decrease in the generated documentation). --- test/lib/helper.py | 13 ++-- test/performance/test_streams.py | 4 +- test/test_base.py | 4 +- test/test_clone.py | 4 +- test/test_commit.py | 16 +++-- test/test_config.py | 12 ++-- test/test_diff.py | 23 ++++--- test/test_docs.py | 26 ++++---- test/test_fun.py | 3 +- test/test_git.py | 22 +++--- test/test_index.py | 49 +++++++------- test/test_installation.py | 6 +- test/test_refs.py | 50 +++++++++----- test/test_remote.py | 39 +++++------ test/test_repo.py | 34 +++++----- test/test_submodule.py | 111 ++++++++++++++++++------------- test/test_util.py | 13 ++-- 17 files changed, 243 insertions(+), 186 deletions(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index f951a6a12..26469ed5d 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -346,17 +346,20 @@ class TestBase(TestCase): """Base class providing default functionality to all tests such as: - Utility functions provided by the TestCase base of the unittest method such as:: + self.fail("todo") self.assertRaises(...) - Class level repository which is considered read-only as it is shared among all test cases in your type. + Access it using:: - self.rorepo # 'ro' stands for read-only + + self.rorepo # 'ro' stands for read-only The rorepo is in fact your current project's git repo. If you refer to specific - shas for your objects, be sure you choose some that are part of the immutable portion - of the project history (so that tests don't fail for others). + shas for your objects, be sure you choose some that are part of the immutable + portion of the project history (so that tests don't fail for others). """ def _small_repo_url(self): @@ -383,8 +386,8 @@ def tearDownClass(cls): def _make_file(self, rela_path, data, repo=None): """ - Create a file at the given path relative to our repository, filled - with the given data. + Create a file at the given path relative to our repository, filled with the + given data. :return: An absolute path to the created file. """ diff --git a/test/performance/test_streams.py b/test/performance/test_streams.py index ba5cbe415..56b5274ec 100644 --- a/test/performance/test_streams.py +++ b/test/performance/test_streams.py @@ -25,8 +25,8 @@ class TestObjDBPerformance(TestBigRepoR): @with_rw_repo("HEAD", bare=True) def test_large_data_streaming(self, rwrepo): - # TODO: This part overlaps with the same file in gitdb.test.performance.test_stream. - # It should be shared if possible. + # TODO: This part overlaps with the same file in + # gitdb.test.performance.test_stream. It should be shared if possible. ldb = LooseObjectDB(osp.join(rwrepo.git_dir, "objects")) for randomize in range(2): diff --git a/test/test_base.py b/test/test_base.py index 693c97f20..ef7486e86 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -135,8 +135,8 @@ def test_add_unicode(self, rw_repo): # https://github.com/gitpython-developers/GitPython/issues/147#issuecomment-68881897 # Therefore, it must be added using the Python implementation. rw_repo.index.add([file_path]) - # However, when the test winds down, rmtree fails to delete this file, which is recognized - # as ??? only. + # However, when the test winds down, rmtree fails to delete this file, which + # is recognized as ??? only. else: # On POSIX, we can just add Unicode files without problems. rw_repo.git.add(rw_repo.working_dir) diff --git a/test/test_clone.py b/test/test_clone.py index dcab7ad6f..be2e6b19b 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -19,8 +19,8 @@ def test_checkout_in_non_empty_dir(self, rw_dir): garbage_file = non_empty_dir / "not-empty" garbage_file.write_text("Garbage!") - # Verify that cloning into the non-empty dir fails while complaining about - # the target directory not being empty/non-existent. + # Verify that cloning into the non-empty dir fails while complaining about the + # target directory not being empty/non-existent. try: self.rorepo.clone(non_empty_dir) except git.GitCommandError as exc: diff --git a/test/test_commit.py b/test/test_commit.py index fddb91c17..5571b9ecb 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -27,8 +27,8 @@ class TestCommitSerialization(TestBase): def assert_commit_serialization(self, rwrepo, commit_id, print_performance_info=False): - """Traverse all commits in the history of commit identified by commit_id and check - if the serialization works. + """Traverse all commits in the history of commit identified by commit_id and + check if the serialization works. :param print_performance_info: If True, we will show how fast we are. """ @@ -317,8 +317,9 @@ def test_count(self): self.assertEqual(self.rorepo.tag("refs/tags/0.1.5").commit.count(), 143) def test_list(self): - # This doesn't work anymore, as we will either attempt getattr with bytes, or compare 20 byte string - # with actual 20 byte bytes. This usage makes no sense anyway. + # This doesn't work anymore, as we will either attempt getattr with bytes, or + # compare 20 byte string with actual 20 byte bytes. This usage makes no sense + # anyway. assert isinstance( Commit.list_items(self.rorepo, "0.1.5", max_count=5)["5117c9c8a4d3af19a9958677e45cda9269de1541"], Commit, @@ -383,8 +384,8 @@ def test_serialization_unicode_support(self): self.assertEqual(cmt.author.name, ncmt.author.name) self.assertEqual(cmt.message, ncmt.message) - # Actually, it can't be printed in a shell as repr wants to have ascii only - # it appears. + # Actually, it can't be printed in a shell as repr wants to have ascii only it + # appears. cmt.author.__repr__() def test_invalid_commit(self): @@ -498,7 +499,8 @@ def test_trailers(self): KEY_2 = "Key" VALUE_2 = "Value with inner spaces" - # Check that the following trailer example is extracted from multiple msg variations. + # Check that the following trailer example is extracted from multiple msg + # variations. TRAILER = f"{KEY_1}: {VALUE_1_1}\n{KEY_2}: {VALUE_2}\n{KEY_1}: {VALUE_1_2}" msgs = [ f"Subject\n\n{TRAILER}\n", diff --git a/test/test_config.py b/test/test_config.py index f493c5672..4843d91eb 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -41,7 +41,7 @@ def _to_memcache(self, file_path): return sio def test_read_write(self): - # writer must create the exact same file as the one read before + # The writer must create the exact same file as the one read before. for filename in ("git_config", "git_config_global"): file_obj = self._to_memcache(fixture_path(filename)) with GitConfigParser(file_obj, read_only=False) as w_config: @@ -56,7 +56,8 @@ def test_read_write(self): self._to_memcache(fixture_path(filename)).getvalue(), ) - # Creating an additional config writer must fail due to exclusive access. + # Creating an additional config writer must fail due to exclusive + # access. with self.assertRaises(IOError): GitConfigParser(file_obj, read_only=False) @@ -91,8 +92,8 @@ def test_includes_order(self): r_config.read() # Enforce reading. # Simple inclusions, again checking them taking precedence. assert r_config.get_value("sec", "var0") == "value0_included" - # This one should take the git_config_global value since included - # values must be considered as soon as they get them. + # This one should take the git_config_global value since included values + # must be considered as soon as they get them. assert r_config.get_value("diff", "tool") == "meld" try: # FIXME: Split this assertion out somehow and mark it xfail (or fix it). @@ -109,7 +110,8 @@ def test_lock_reentry(self, rw_dir): # Entering again locks the file again... with gcp as cw: cw.set_value("include", "some_other_value", "b") - # ...so creating an additional config writer must fail due to exclusive access. + # ...so creating an additional config writer must fail due to exclusive + # access. with self.assertRaises(IOError): GitConfigParser(fpl, read_only=False) # but work when the lock is removed diff --git a/test/test_diff.py b/test/test_diff.py index 87f92f5d1..cdd473f7f 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -271,18 +271,18 @@ def test_diff_unsafe_paths(self): self.assertEqual(res[10].b_rawpath, b"path/\x80-invalid-unicode-path.txt") # The "Moves" - # NOTE: The path prefixes a/ and b/ here are legit! We're actually - # verifying that it's not "a/a/" that shows up, see the fixture data. - self.assertEqual(res[11].a_path, "a/with spaces") # NOTE: path a/ here legit! - self.assertEqual(res[11].b_path, "b/with some spaces") # NOTE: path b/ here legit! + # NOTE: The path prefixes "a/" and "b/" here are legit! We're actually verifying + # that it's not "a/a/" that shows up; see the fixture data. + self.assertEqual(res[11].a_path, "a/with spaces") # NOTE: path "a/"" legit! + self.assertEqual(res[11].b_path, "b/with some spaces") # NOTE: path "b/"" legit! self.assertEqual(res[12].a_path, "a/ending in a space ") self.assertEqual(res[12].b_path, "b/ending with space ") self.assertEqual(res[13].a_path, 'a/"with-quotes"') self.assertEqual(res[13].b_path, 'b/"with even more quotes"') def test_diff_patch_format(self): - # Test all of the 'old' format diffs for completeness - it should at least - # be able to deal with it. + # Test all of the 'old' format diffs for completeness - it should at least be + # able to deal with it. fixtures = ( "diff_2", "diff_2f", @@ -345,8 +345,9 @@ def test_diff_submodule(self): repo.create_tag("2") diff = repo.commit("1").diff(repo.commit("2"))[0] - # If diff is unable to find the commit hashes (looks in wrong repo) the *_blob.size - # property will be a string containing exception text, an int indicates success. + # If diff is unable to find the commit hashes (looks in wrong repo) the + # *_blob.size property will be a string containing exception text, an int + # indicates success. self.assertIsInstance(diff.a_blob.size, int) self.assertIsInstance(diff.b_blob.size, int) @@ -392,9 +393,9 @@ def test_diff_interface(self): # END for each other side # END for each commit - # Assert that we could always find at least one instance of the members we - # can iterate in the diff index - if not this indicates its not working correctly - # or our test does not span the whole range of possibilities. + # Assert that we could always find at least one instance of the members we can + # iterate in the diff index - if not this indicates its not working correctly or + # our test does not span the whole range of possibilities. for key, value in assertion_map.items(): self.assertIsNotNone(value, "Did not find diff for %s" % key) # END for each iteration type diff --git a/test/test_docs.py b/test/test_docs.py index 48922eba8..409f66bb3 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -19,8 +19,9 @@ class Tutorials(TestBase): def tearDown(self): gc.collect() - # ACTUALLY skipped by git.util.rmtree (in local onerror function), from the last call to it via - # git.objects.submodule.base.Submodule.remove (at "handle separate bare repository"), line 1062. + # ACTUALLY skipped by git.util.rmtree (in local onerror function), from the last + # call to it via git.objects.submodule.base.Submodule.remove + # (at "handle separate bare repository"), line 1062. # # @skipIf(HIDE_WINDOWS_KNOWN_ERRORS, # "FIXME: helper.wrapper fails with: PermissionError: [WinError 5] Access is denied: " @@ -31,8 +32,8 @@ def test_init_repo_object(self, rw_dir): from git import Repo # rorepo is a Repo instance pointing to the git-python repository. - # For all you know, the first argument to Repo is a path to the repository - # you want to work with. + # For all you know, the first argument to Repo is a path to the repository you + # want to work with. repo = Repo(self.rorepo.working_tree_dir) assert not repo.bare # ![1-test_init_repo_object] @@ -149,8 +150,8 @@ def update(self, op_code, cur_count, max_count=None, message=""): assert origin.exists() for fetch_info in origin.fetch(progress=MyProgressPrinter()): print("Updated %s to %s" % (fetch_info.ref, fetch_info.commit)) - # Create a local branch at the latest fetched master. We specify the name statically, but you have all - # information to do it programmatically as well. + # Create a local branch at the latest fetched master. We specify the name + # statically, but you have all information to do it programmatically as well. bare_master = bare_repo.create_head("master", origin.refs.master) bare_repo.head.set_reference(bare_master) assert not bare_repo.delete_remote(origin).exists() @@ -188,9 +189,9 @@ def update(self, op_code, cur_count, max_count=None, message=""): # submodules # [14-test_init_repo_object] - # Create a new submodule and check it out on the spot, setup to track master branch of `bare_repo`. - # As our GitPython repository has submodules already that point to GitHub, make sure we don't - # interact with them. + # Create a new submodule and check it out on the spot, setup to track master + # branch of `bare_repo`. As our GitPython repository has submodules already that + # point to GitHub, make sure we don't interact with them. for sm in cloned_repo.submodules: assert not sm.remove().exists() # after removal, the sm doesn't exist anymore sm = cloned_repo.create_submodule("mysubrepo", "path/to/subrepo", url=bare_repo.git_dir, branch="master") @@ -424,8 +425,8 @@ def test_references_and_objects(self, rw_dir): with origin.config_writer as cw: cw.set("pushurl", "other_url") - # Please note that in Python 2, writing origin.config_writer.set(...) is totally safe. - # In py3 __del__ calls can be delayed, thus not writing changes in time. + # Please note that in Python 2, writing origin.config_writer.set(...) is totally + # safe. In py3 __del__ calls can be delayed, thus not writing changes in time. # ![26-test_references_and_objects] # [27-test_references_and_objects] @@ -462,7 +463,8 @@ def test_references_and_objects(self, rw_dir): # ![29-test_references_and_objects] # [30-test_references_and_objects] - # Check out the branch using git-checkout. It will fail as the working tree appears dirty. + # Check out the branch using git-checkout. + # It will fail as the working tree appears dirty. self.assertRaises(git.GitCommandError, repo.heads.master.checkout) repo.heads.past_branch.checkout() # ![30-test_references_and_objects] diff --git a/test/test_fun.py b/test/test_fun.py index 566bc9aae..2d30d355a 100644 --- a/test/test_fun.py +++ b/test/test_fun.py @@ -35,7 +35,8 @@ def _assert_index_entries(self, entries, trees): # END assert entry matches fully def test_aggressive_tree_merge(self): - # Head tree with additions, removals and modification compared to its predecessor. + # Head tree with additions, removals and modification compared to its + # predecessor. odb = self.rorepo.odb HC = self.rorepo.commit("6c1faef799095f3990e9970bc2cb10aa0221cf9c") H = HC.tree diff --git a/test/test_git.py b/test/test_git.py index 97e21cad4..e1a8bda5e 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -253,21 +253,27 @@ def test_it_avoids_upcasing_unrelated_environment_variable_names(self): if old_name == old_name.upper(): raise RuntimeError("test bug or strange locale: old_name invariant under upcasing") - # Step 1: Set the environment variable in this parent process. Because os.putenv is a thin - # wrapper around a system API, os.environ never sees the variable in this parent - # process, so the name is not upcased even on Windows. + # Step 1 + # + # Set the environment variable in this parent process. Because os.putenv is a + # thin wrapper around a system API, os.environ never sees the variable in this + # parent process, so the name is not upcased even on Windows. os.putenv(old_name, "1") - # Step 2: Create the child process that inherits the environment variable. The child uses - # GitPython, and we are testing that it passes the variable with the exact original - # name to its own child process (the grandchild). + # Step 2 + # + # Create the child process that inherits the environment variable. The child + # uses GitPython, and we are testing that it passes the variable with the exact + # original name to its own child process (the grandchild). cmdline = [ sys.executable, fixture_path("env_case.py"), # Contains steps 3 and 4. self.rorepo.working_dir, old_name, ] - pair_text = subprocess.check_output(cmdline, shell=False, text=True) # Run steps 3 and 4. + + # Run steps 3 and 4. + pair_text = subprocess.check_output(cmdline, shell=False, text=True) new_name = pair_text.split("=")[0] self.assertEqual(new_name, old_name) @@ -668,7 +674,7 @@ def test_successful_default_refresh_invalidates_cached_version_info(self): # as unintended shell expansions can occur, and is deprecated. Instead, # use a custom command, by setting the GIT_PYTHON_GIT_EXECUTABLE # environment variable to git.cmd or by passing git.cmd's full path to - # git.refresh. Or wrap the script with a .exe shim. + # git.refresh. Or wrap the script with a .exe shim.) stack.enter_context(mock.patch.object(Git, "USE_SHELL", True)) new_git = Git() diff --git a/test/test_index.py b/test/test_index.py index fd62bb893..fa64b82a2 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -219,8 +219,7 @@ def _fprogress(self, path, done, item): self._fprogress_map[path] = curval + 1 def _fprogress_add(self, path, done, item): - """Called as progress func - we keep track of the proper - call order""" + """Called as progress func - we keep track of the proper call order.""" assert item is not None self._fprogress(path, done, item) @@ -385,17 +384,17 @@ def test_index_merge_tree(self, rw_repo): # FAKE MERGE ############# - # Add a change with a NULL sha that should conflict with next_commit. We - # pretend there was a change, but we do not even bother adding a proper - # sha for it (which makes things faster of course). + # Add a change with a NULL sha that should conflict with next_commit. We pretend + # there was a change, but we do not even bother adding a proper sha for it + # (which makes things faster of course). manifest_fake_entry = BaseIndexEntry((manifest_entry[0], b"\0" * 20, 0, manifest_entry[3])) # Try write flag. self._assert_entries(rw_repo.index.add([manifest_fake_entry], write=False)) - # Add actually resolves the null-hex-sha for us as a feature, but we can - # edit the index manually. + # Add actually resolves the null-hex-sha for us as a feature, but we can edit + # the index manually. assert rw_repo.index.entries[manifest_key].binsha != Object.NULL_BIN_SHA - # We must operate on the same index for this! It's a bit problematic as - # it might confuse people. + # We must operate on the same index for this! It's a bit problematic as it might + # confuse people. index = rw_repo.index index.entries[manifest_key] = IndexEntry.from_base(manifest_fake_entry) index.write() @@ -404,19 +403,20 @@ def test_index_merge_tree(self, rw_repo): # Write an unchanged index (just for the fun of it). rw_repo.index.write() - # A three way merge would result in a conflict and fails as the command will - # not overwrite any entries in our index and hence leave them unmerged. This is + # A three way merge would result in a conflict and fails as the command will not + # overwrite any entries in our index and hence leave them unmerged. This is # mainly a protection feature as the current index is not yet in a tree. self.assertRaises(GitCommandError, index.merge_tree, next_commit, base=parent_commit) - # The only way to get the merged entries is to safe the current index away into a tree, - # which is like a temporary commit for us. This fails as well as the NULL sha does not - # have a corresponding object. + # The only way to get the merged entries is to safe the current index away into + # a tree, which is like a temporary commit for us. This fails as well as the + # NULL sha does not have a corresponding object. # NOTE: missing_ok is not a kwarg anymore, missing_ok is always true. # self.assertRaises(GitCommandError, index.write_tree) - # If missing objects are okay, this would work though (they are always okay now). - # As we can't read back the tree with NULL_SHA, we rather set it to something else. + # If missing objects are okay, this would work though (they are always okay + # now). As we can't read back the tree with NULL_SHA, we rather set it to + # something else. index.entries[manifest_key] = IndexEntry(manifest_entry[:1] + (hex_to_bin("f" * 40),) + manifest_entry[2:]) tree = index.write_tree() @@ -428,7 +428,7 @@ def test_index_merge_tree(self, rw_repo): @with_rw_repo("0.1.6") def test_index_file_diffing(self, rw_repo): - # Default Index instance points to our index. + # Default IndexFile instance points to our index. index = IndexFile(rw_repo) assert index.path is not None assert len(index.entries) @@ -439,8 +439,8 @@ def test_index_file_diffing(self, rw_repo): # Could sha it, or check stats. # Test diff. - # Resetting the head will leave the index in a different state, and the - # diff will yield a few changes. + # Resetting the head will leave the index in a different state, and the diff + # will yield a few changes. cur_head_commit = rw_repo.head.reference.commit rw_repo.head.reset("HEAD~6", index=True, working_tree=False) @@ -956,10 +956,10 @@ def test_index_new(self): @with_rw_repo("HEAD", bare=True) def test_index_bare_add(self, rw_bare_repo): - # Something is wrong after cloning to a bare repo, reading the - # property rw_bare_repo.working_tree_dir will return '/tmp' - # instead of throwing the Exception we are expecting. This is - # a quick hack to make this test fail when expected. + # Something is wrong after cloning to a bare repo, reading the property + # rw_bare_repo.working_tree_dir will return '/tmp' instead of throwing the + # Exception we are expecting. This is a quick hack to make this test fail when + # expected. assert rw_bare_repo.working_tree_dir is None assert rw_bare_repo.bare contents = b"This is a BytesIO file" @@ -984,7 +984,8 @@ def test_index_bare_add(self, rw_bare_repo): @with_rw_directory def test_add_utf8P_path(self, rw_dir): - # NOTE: fp is not a Unicode object in Python 2 (which is the source of the problem). + # NOTE: fp is not a Unicode object in Python 2 + # (which is the source of the problem). fp = osp.join(rw_dir, "ø.txt") with open(fp, "wb") as fs: fs.write("content of ø".encode("utf-8")) diff --git a/test/test_installation.py b/test/test_installation.py index 15ed5b13b..ae6472e98 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -46,9 +46,9 @@ def test_installation(self, rw_dir): msg=result.stderr or result.stdout or "Dependencies not installed", ) - # Even IF gitdb or any other dependency is supplied during development - # by inserting its location into PYTHONPATH or otherwise patched into - # sys.path, make sure it is not wrongly inserted as the *first* entry. + # Even IF gitdb or any other dependency is supplied during development by + # inserting its location into PYTHONPATH or otherwise patched into sys.path, + # make sure it is not wrongly inserted as the *first* entry. result = subprocess.run( [venv.python, "-c", "import sys; import git; print(sys.path)"], stdout=subprocess.PIPE, diff --git a/test/test_refs.py b/test/test_refs.py index 2656ceab1..28db70c6e 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -245,8 +245,8 @@ def test_head_reset(self, rw_repo): cur_head.reset(new_head_commit) rw_repo.index.checkout(["lib"], force=True) - # Now that we have a write write repo, change the HEAD reference - it's - # like "git-reset --soft". + # Now that we have a write write repo, change the HEAD reference - it's like + # "git-reset --soft". heads = rw_repo.heads assert heads for head in heads: @@ -349,8 +349,8 @@ def test_head_reset(self, rw_repo): for remote in remotes: refs = remote.refs - # If a HEAD exists, it must be deleted first. Otherwise it might - # end up pointing to an invalid ref it the ref was deleted before. + # If a HEAD exists, it must be deleted first. Otherwise it might end up + # pointing to an invalid ref it the ref was deleted before. remote_head_name = "HEAD" if remote_head_name in refs: RemoteReference.delete(rw_repo, refs[remote_head_name]) @@ -383,7 +383,7 @@ def test_head_reset(self, rw_repo): # Setting a non-commit as commit fails, but succeeds as object. head_tree = head.commit.tree self.assertRaises(ValueError, setattr, head, "commit", head_tree) - assert head.commit == old_commit # and the ref did not change + assert head.commit == old_commit # And the ref did not change. # We allow heads to point to any object. head.object = head_tree assert head.object == head_tree @@ -492,8 +492,8 @@ def test_head_reset(self, rw_repo): # Would raise if the symref wouldn't have been deleted (probably). symref = SymbolicReference.create(rw_repo, symref_path, cur_head.reference) - # Test symbolic references which are not at default locations like HEAD - # or FETCH_HEAD - they may also be at spots in refs of course. + # Test symbolic references which are not at default locations like HEAD or + # FETCH_HEAD - they may also be at spots in refs of course. symbol_ref_path = "refs/symbol_ref" symref = SymbolicReference(rw_repo, symbol_ref_path) assert symref.path == symbol_ref_path @@ -525,14 +525,13 @@ def test_head_reset(self, rw_repo): assert active_branch in heads assert rw_repo.tags - # We should be able to iterate all symbolic refs as well - in that case - # we should expect only symbolic references to be returned. + # We should be able to iterate all symbolic refs as well - in that case we + # should expect only symbolic references to be returned. for symref in SymbolicReference.iter_items(rw_repo): assert not symref.is_detached - # When iterating references, we can get references and symrefs - # when deleting all refs, I'd expect them to be gone! Even from - # the packed ones. + # When iterating references, we can get references and symrefs when deleting all + # refs, I'd expect them to be gone! Even from the packed ones. # For this to work, we must not be on any branch. rw_repo.head.reference = rw_repo.head.commit deleted_refs = set() @@ -577,9 +576,9 @@ def test_head_reset(self, rw_repo): self.assertRaises(ValueError, setattr, ref, "commit", "nonsense") assert not ref.is_valid() - # I am sure I had my reason to make it a class method at first, but - # now it doesn't make so much sense anymore, want an instance method as well - # See http://byronimo.lighthouseapp.com/projects/51787-gitpython/tickets/27 + # I am sure I had my reason to make it a class method at first, but now it + # doesn't make so much sense anymore, want an instance method as well. See: + # http://byronimo.lighthouseapp.com/projects/51787-gitpython/tickets/27 Reference.delete(ref.repo, ref.path) assert not ref.is_valid() @@ -619,8 +618,8 @@ def test_reflog(self): def test_refs_outside_repo(self): # Create a file containing a valid reference outside the repository. Attempting - # to access it should raise an exception, due to it containing a parent directory - # reference ('..'). This tests for CVE-2023-41040. + # to access it should raise an exception, due to it containing a parent + # directory reference ('..'). This tests for CVE-2023-41040. git_dir = Path(self.rorepo.git_dir) repo_parent_dir = git_dir.parent.parent with tempfile.NamedTemporaryFile(dir=repo_parent_dir) as ref_file: @@ -630,37 +629,52 @@ def test_refs_outside_repo(self): self.assertRaises(BadName, self.rorepo.commit, f"../../{ref_file_name}") def test_validity_ref_names(self): + """Ensure ref names are checked for validity. + + This is based on the rules specified in: + https://git-scm.com/docs/git-check-ref-format/#_description + """ check_ref = SymbolicReference._check_ref_name_valid - # Based on the rules specified in https://git-scm.com/docs/git-check-ref-format/#_description. + # Rule 1 self.assertRaises(ValueError, check_ref, ".ref/begins/with/dot") self.assertRaises(ValueError, check_ref, "ref/component/.begins/with/dot") self.assertRaises(ValueError, check_ref, "ref/ends/with/a.lock") self.assertRaises(ValueError, check_ref, "ref/component/ends.lock/with/period_lock") + # Rule 2 check_ref("valid_one_level_refname") + # Rule 3 self.assertRaises(ValueError, check_ref, "ref/contains/../double/period") + # Rule 4 for c in " ~^:": self.assertRaises(ValueError, check_ref, f"ref/contains/invalid{c}/character") for code in range(0, 32): self.assertRaises(ValueError, check_ref, f"ref/contains/invalid{chr(code)}/ASCII/control_character") self.assertRaises(ValueError, check_ref, f"ref/contains/invalid{chr(127)}/ASCII/control_character") + # Rule 5 for c in "*?[": self.assertRaises(ValueError, check_ref, f"ref/contains/invalid{c}/character") + # Rule 6 self.assertRaises(ValueError, check_ref, "/ref/begins/with/slash") self.assertRaises(ValueError, check_ref, "ref/ends/with/slash/") self.assertRaises(ValueError, check_ref, "ref/contains//double/slash/") + # Rule 7 self.assertRaises(ValueError, check_ref, "ref/ends/with/dot.") + # Rule 8 self.assertRaises(ValueError, check_ref, "ref/contains@{/at_brace") + # Rule 9 self.assertRaises(ValueError, check_ref, "@") + # Rule 10 self.assertRaises(ValueError, check_ref, "ref/contain\\s/backslash") + # Valid reference name should not raise. check_ref("valid/ref/name") diff --git a/test/test_remote.py b/test/test_remote.py index df6034326..c0bd11f80 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -294,11 +294,11 @@ def get_info(res, remote, name): # Provoke to receive actual objects to see what kind of output we have to # expect. For that we need a remote transport protocol. - # Create a new UN-shared repo and fetch into it after we pushed a change - # to the shared repo. + # Create a new UN-shared repo and fetch into it after we pushed a change to the + # shared repo. other_repo_dir = tempfile.mktemp("other_repo") - # Must clone with a local path for the repo implementation not to freak out - # as it wants local paths only (which I can understand). + # Must clone with a local path for the repo implementation not to freak out as + # it wants local paths only (which I can understand). other_repo = remote_repo.clone(other_repo_dir, shared=False) remote_repo_url = osp.basename(remote_repo.git_dir) # git-daemon runs with appropriate `--base-path`. remote_repo_url = Git.polish_url("git://localhost:%s/%s" % (GIT_DAEMON_PORT, remote_repo_url)) @@ -317,10 +317,10 @@ def get_info(res, remote, name): self._commit_random_file(rw_repo) remote.push(rw_repo.head.reference) - # Here I would expect to see remote-information about packing - # objects and so on. Unfortunately, this does not happen - # if we are redirecting the output - git explicitly checks for this - # and only provides progress information to ttys. + # Here I would expect to see remote-information about packing objects and so + # on. Unfortunately, this does not happen if we are redirecting the output - + # git explicitly checks for this and only provides progress information to + # ttys. res = fetch_and_test(other_origin) finally: rmtree(other_repo_dir) @@ -333,8 +333,8 @@ def _assert_push_and_pull(self, remote, rw_repo, remote_repo): try: lhead.reference = rw_repo.heads.master except AttributeError: - # If the author is on a non-master branch, the clones might not have - # a local master yet. We simply create it. + # If the author is on a non-master branch, the clones might not have a local + # master yet. We simply create it. lhead.reference = rw_repo.create_head("master") # END master handling lhead.reset(remote.refs.master, working_tree=True) @@ -488,8 +488,8 @@ def test_base(self, rw_repo, remote_repo): self._assert_push_and_pull(remote, rw_repo, remote_repo) # FETCH TESTING - # Only for remotes - local cases are the same or less complicated - # as additional progress information will never be emitted. + # Only for remotes - local cases are the same or less complicated as + # additional progress information will never be emitted. if remote.name == "daemon_origin": self._do_test_fetch(remote, rw_repo, remote_repo, kill_after_timeout=10.0) ran_fetch_test = True @@ -508,7 +508,8 @@ def test_base(self, rw_repo, remote_repo): # Verify we can handle prunes when fetching. # stderr lines look like this: x [deleted] (none) -> origin/experiment-2012 # These should just be skipped. - # If we don't have a manual checkout, we can't actually assume there are any non-master branches. + # If we don't have a manual checkout, we can't actually assume there are any + # non-master branches. remote_repo.create_head("myone_for_deletion") # Get the branch - to be pruned later origin.fetch() @@ -812,8 +813,8 @@ def test_fetch_unsafe_url_allowed(self, rw_repo): "fd::17/foo", ] for url in urls: - # The URL will be allowed into the command, but the command will - # fail since we don't have that protocol enabled in the Git config file. + # The URL will be allowed into the command, but the command will fail + # since we don't have that protocol enabled in the Git config file. with self.assertRaises(GitCommandError): remote.fetch(url, allow_unsafe_protocols=True) assert not tmp_file.exists() @@ -880,8 +881,8 @@ def test_pull_unsafe_url_allowed(self, rw_repo): "fd::17/foo", ] for url in urls: - # The URL will be allowed into the command, but the command will - # fail since we don't have that protocol enabled in the Git config file. + # The URL will be allowed into the command, but the command will fail + # since we don't have that protocol enabled in the Git config file. with self.assertRaises(GitCommandError): remote.pull(url, allow_unsafe_protocols=True) assert not tmp_file.exists() @@ -948,8 +949,8 @@ def test_push_unsafe_url_allowed(self, rw_repo): "fd::17/foo", ] for url in urls: - # The URL will be allowed into the command, but the command will - # fail since we don't have that protocol enabled in the Git config file. + # The URL will be allowed into the command, but the command will fail + # since we don't have that protocol enabled in the Git config file. with self.assertRaises(GitCommandError): remote.push(url, allow_unsafe_protocols=True) assert not tmp_file.exists() diff --git a/test/test_repo.py b/test/test_repo.py index 465bb2574..30a44b6c1 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -543,8 +543,8 @@ def test_init(self): try: rmtree(clone_path) except OSError: - # When relative paths are used, the clone may actually be inside - # of the parent directory. + # When relative paths are used, the clone may actually be inside of + # the parent directory. pass # END exception handling @@ -556,8 +556,8 @@ def test_init(self): try: rmtree(clone_path) except OSError: - # When relative paths are used, the clone may actually be inside - # of the parent directory. + # When relative paths are used, the clone may actually be inside of + # the parent directory. pass # END exception handling @@ -832,8 +832,8 @@ def test_config_level_paths(self): assert self.rorepo._get_config_path(config_level) def test_creation_deletion(self): - # Just a very quick test to assure it generally works. There are - # specialized cases in the test_refs module. + # Just a very quick test to assure it generally works. There are specialized + # cases in the test_refs module. head = self.rorepo.create_head("new_head", "HEAD~1") self.rorepo.delete_head(head) @@ -1027,7 +1027,8 @@ def test_rev_parse(self): num_resolved += 1 except (BadName, BadObject): print("failed on %s" % path_section) - # This is fine if we have something like 112, which belongs to remotes/rname/merge-requests/112. + # This is fine if we have something like 112, which belongs to + # remotes/rname/merge-requests/112. # END exception handling # END for each token if ref_no == 3 - 1: @@ -1149,7 +1150,7 @@ def test_submodule_update(self, rwrepo): ) self.assertIsInstance(sm, Submodule) - # NOTE: the rest of this functionality is tested in test_submodule. + # NOTE: The rest of this functionality is tested in test_submodule. @with_rw_repo("HEAD") def test_git_file(self, rwrepo): @@ -1178,8 +1179,9 @@ def last_commit(repo, rev, path): # This is based on this comment: # https://github.com/gitpython-developers/GitPython/issues/60#issuecomment-23558741 # And we expect to set max handles to a low value, like 64. - # You should set ulimit -n X, see .travis.yml - # The loops below would easily create 500 handles if these would leak (4 pipes + multiple mapped files). + # You should set ulimit -n X. See .travis.yml. + # The loops below would easily create 500 handles if these would leak + # (4 pipes + multiple mapped files). for _ in range(64): for repo_type in (GitCmdObjectDB, GitDB): repo = Repo(self.rorepo.working_tree_dir, odbt=repo_type) @@ -1200,8 +1202,8 @@ def test_empty_repo(self, rw_dir): self.assertEqual(r.active_branch.name, "master") assert not r.active_branch.is_valid(), "Branch is yet to be born" - # Actually, when trying to create a new branch without a commit, git itself fails. - # We should, however, not fail ungracefully. + # Actually, when trying to create a new branch without a commit, git itself + # fails. We should, however, not fail ungracefully. self.assertRaises(BadName, r.create_head, "foo") self.assertRaises(BadName, r.create_head, "master") # It's expected to not be able to access a tree @@ -1315,13 +1317,13 @@ def test_git_work_tree_dotgit(self, rw_dir): repo = Repo(worktree_path) self.assertIsInstance(repo, Repo) - # This ensures we're able to actually read the refs in the tree, which - # means we can read commondir correctly. + # This ensures we're able to actually read the refs in the tree, which means we + # can read commondir correctly. commit = repo.head.commit self.assertIsInstance(commit, Object) - # This ensures we can read the remotes, which confirms we're reading - # the config correctly. + # This ensures we can read the remotes, which confirms we're reading the config + # correctly. origin = repo.remotes.origin self.assertIsInstance(origin, Remote) diff --git a/test/test_submodule.py b/test/test_submodule.py index 993f6b57e..68164729b 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -94,13 +94,15 @@ def _do_base_tests(self, rwrepo): # The module is not checked-out yet. self.assertRaises(InvalidGitRepositoryError, sm.module) - # ...which is why we can't get the branch either - it points into the module() repository. + # ...which is why we can't get the branch either - it points into the module() + # repository. self.assertRaises(InvalidGitRepositoryError, getattr, sm, "branch") # branch_path works, as it's just a string. assert isinstance(sm.branch_path, str) - # Some commits earlier we still have a submodule, but it's at a different commit. + # Some commits earlier we still have a submodule, but it's at a different + # commit. smold = next(Submodule.iter_items(rwrepo, self.k_subm_changed)) assert smold.binsha != sm.binsha assert smold != sm # the name changed @@ -141,11 +143,12 @@ def _do_base_tests(self, rwrepo): smold.set_parent_commit(self.k_subm_changed + "~1") assert smold.binsha != sm.binsha - # Raises if the sm didn't exist in new parent - it keeps its - # parent_commit unchanged. + # Raises if the sm didn't exist in new parent - it keeps its parent_commit + # unchanged. self.assertRaises(ValueError, smold.set_parent_commit, self.k_no_subm_tag) - # TEST TODO: If a path is in the .gitmodules file, but not in the index, it raises. + # TODO: Test that, if a path is in the .gitmodules file, but not in the index, + # then it raises. # TEST UPDATE ############## @@ -196,8 +199,8 @@ def _do_base_tests(self, rwrepo): # INTERLEAVE ADD TEST ##################### - # url must match the one in the existing repository (if submodule name suggests a new one) - # or we raise. + # url must match the one in the existing repository (if submodule name + # suggests a new one) or we raise. self.assertRaises( ValueError, Submodule.add, @@ -228,7 +231,8 @@ def _do_base_tests(self, rwrepo): assert not csm.module_exists() csm_repopath = csm.path - # Adjust the path of the submodules module to point to the local destination. + # Adjust the path of the submodules module to point to the local + # destination. new_csmclone_path = Git.polish_url(osp.join(self.rorepo.working_tree_dir, sm.path, csm.path)) with csm.config_writer() as writer: writer.set_value("url", new_csmclone_path) @@ -249,7 +253,8 @@ def _do_base_tests(self, rwrepo): # This flushed in a sub-submodule. assert len(list(rwrepo.iter_submodules())) == 2 - # Reset both heads to the previous version, verify that to_latest_revision works. + # Reset both heads to the previous version, verify that to_latest_revision + # works. smods = (sm.module(), csm.module()) for repo in smods: repo.head.reset("HEAD~2", working_tree=1) @@ -296,8 +301,8 @@ def _do_base_tests(self, rwrepo): # Must delete something. self.assertRaises(ValueError, csm.remove, module=False, configuration=False) - # module() is supposed to point to gitdb, which has a child-submodule whose URL is still pointing - # to GitHub. To save time, we will change it to: + # module() is supposed to point to gitdb, which has a child-submodule whose + # URL is still pointing to GitHub. To save time, we will change it to: csm.set_parent_commit(csm.repo.head.commit) with csm.config_writer() as cw: cw.set_value("url", self._small_repo_url()) @@ -399,8 +404,8 @@ def _do_base_tests(self, rwrepo): rwrepo.index.commit("my submod commit") assert len(rwrepo.submodules) == 2 - # Needs update, as the head changed. It thinks it's in the history - # of the repo otherwise. + # Needs update, as the head changed. + # It thinks it's in the history of the repo otherwise. nsm.set_parent_commit(rwrepo.head.commit) osm.set_parent_commit(rwrepo.head.commit) @@ -434,7 +439,8 @@ def _do_base_tests(self, rwrepo): # REMOVE 'EM ALL ################ - # If a submodule's repo has no remotes, it can't be added without an explicit url. + # If a submodule's repo has no remotes, it can't be added without an + # explicit url. osmod = osm.module() osm.remove(module=False) @@ -510,7 +516,8 @@ def test_root_module(self, rwrepo): # TEST UPDATE ############# - # Set up a commit that removes existing, adds new and modifies existing submodules. + # Set up a commit that removes existing, adds new and modifies existing + # submodules. rm = RootModule(rwrepo) assert len(rm.children()) == 1 @@ -534,13 +541,15 @@ def test_root_module(self, rwrepo): sm.update(recursive=False) assert sm.module_exists() with sm.config_writer() as writer: - writer.set_value("path", fp) # Change path to something with prefix AFTER url change. + # Change path to something with prefix AFTER url change. + writer.set_value("path", fp) - # Update doesn't fail, because list_items ignores the wrong path in such situations. + # Update doesn't fail, because list_items ignores the wrong path in such + # situations. rm.update(recursive=False) - # Move it properly - doesn't work as it its path currently points to an indexentry - # which doesn't exist (move it to some path, it doesn't matter here). + # Move it properly - doesn't work as it its path currently points to an + # indexentry which doesn't exist (move it to some path, it doesn't matter here). self.assertRaises(InvalidGitRepositoryError, sm.move, pp) # Reset the path(cache) to where it was, now it works. sm.path = prep @@ -588,23 +597,27 @@ def test_root_module(self, rwrepo): rm.update(recursive=False, dry_run=True, force_remove=True) assert osp.isdir(smp) - # When removing submodules, we may get new commits as nested submodules are auto-committing changes - # to allow deletions without force, as the index would be dirty otherwise. + # When removing submodules, we may get new commits as nested submodules are + # auto-committing changes to allow deletions without force, as the index would + # be dirty otherwise. # QUESTION: Why does this seem to work in test_git_submodule_compatibility() ? self.assertRaises(InvalidGitRepositoryError, rm.update, recursive=False, force_remove=False) rm.update(recursive=False, force_remove=True) assert not osp.isdir(smp) - # 'Apply work' to the nested submodule and ensure this is not removed/altered during updates - # Need to commit first, otherwise submodule.update wouldn't have a reason to change the head. + # 'Apply work' to the nested submodule and ensure this is not removed/altered + # during updates. We need to commit first, otherwise submodule.update wouldn't + # have a reason to change the head. touch(osp.join(nsm.module().working_tree_dir, "new-file")) - # We cannot expect is_dirty to even run as we wouldn't reset a head to the same location. + # We cannot expect is_dirty to even run as we wouldn't reset a head to the same + # location. assert nsm.module().head.commit.hexsha == nsm.hexsha nsm.module().index.add([nsm]) nsm.module().index.commit("added new file") rm.update(recursive=False, dry_run=True, progress=prog) # Would not change head, and thus doesn't fail. - # Everything we can do from now on will trigger the 'future' check, so no is_dirty() check will even run. - # This would only run if our local branch is in the past and we have uncommitted changes. + # Everything we can do from now on will trigger the 'future' check, so no + # is_dirty() check will even run. This would only run if our local branch is in + # the past and we have uncommitted changes. prev_commit = nsm.module().head.commit rm.update(recursive=False, dry_run=False, progress=prog) @@ -616,8 +629,8 @@ def test_root_module(self, rwrepo): # Change url... # ============= - # ...to the first repository. This way we have a fast checkout, and a completely different - # repository at the different url. + # ...to the first repository. This way we have a fast checkout, and a completely + # different repository at the different url. nsm.set_parent_commit(csmremoved) nsmurl = Git.polish_url(osp.join(self.rorepo.working_tree_dir, rsmsp[0])) with nsm.config_writer() as writer: @@ -637,16 +650,15 @@ def test_root_module(self, rwrepo): assert len(rwrepo.submodules) == 1 assert not rwrepo.submodules[0].children()[0].module_exists(), "nested submodule should not be checked out" - # Add the submodule's changed commit to the index, which is what the - # user would do. - # Beforehand, update our instance's binsha with the new one. + # Add the submodule's changed commit to the index, which is what the user would + # do. Beforehand, update our instance's binsha with the new one. nsm.binsha = nsm.module().head.commit.binsha rwrepo.index.add([nsm]) # Change branch. # ============== - # We only have one branch, so we switch to a virtual one, and back - # to the current one to trigger the difference. + # We only have one branch, so we switch to a virtual one, and back to the + # current one to trigger the difference. cur_branch = nsm.branch nsmm = nsm.module() prev_commit = nsmm.head.commit @@ -808,8 +820,8 @@ def test_git_submodules_and_add_sm_with_new_commit(self, rwdir): smm.git.add(Git.polish_url(fp)) smm.git.commit(m="new file added") - # Submodules are retrieved from the current commit's tree, therefore we can't really get a new submodule - # object pointing to the new submodule commit. + # Submodules are retrieved from the current commit's tree, therefore we can't + # really get a new submodule object pointing to the new submodule commit. sm_too = parent.submodules["module_moved"] assert parent.head.commit.tree[sm.path].binsha == sm.binsha assert sm_too.binsha == sm.binsha, "cached submodule should point to the same commit as updated one" @@ -848,8 +860,9 @@ def assert_exists(sm, value=True): # END assert_exists - # As git is backwards compatible itself, it would still recognize what we do here... unless we really - # muss it up. That's the only reason why the test is still here... + # As git is backwards compatible itself, it would still recognize what we do + # here... unless we really muss it up. That's the only reason why the test is + # still here... assert len(parent.git.submodule().splitlines()) == 1 module_repo_path = osp.join(sm.module().working_tree_dir, ".git") @@ -885,7 +898,8 @@ def assert_exists(sm, value=True): assert_exists(csm) # Rename nested submodule. - # This name would move itself one level deeper - needs special handling internally. + # This name would move itself one level deeper - needs special handling + # internally. new_name = csm.name + "/mine" assert csm.rename(new_name).name == new_name assert_exists(csm) @@ -1011,13 +1025,15 @@ def test_branch_renames(self, rw_dir): sm_source_repo.index.add([new_file]) sm.repo.index.commit("added new file") - # Change designated submodule checkout branch to the new upstream feature branch. + # Change designated submodule checkout branch to the new upstream feature + # branch. with sm.config_writer() as smcw: smcw.set_value("branch", sm_fb.name) assert sm.repo.is_dirty(index=True, working_tree=False) sm.repo.index.commit("changed submodule branch to '%s'" % sm_fb) - # Verify submodule update with feature branch that leaves currently checked out branch in it's past. + # Verify submodule update with feature branch that leaves currently checked out + # branch in it's past. sm_mod = sm.module() prev_commit = sm_mod.commit() assert sm_mod.head.ref.name == "master" @@ -1029,7 +1045,8 @@ def test_branch_renames(self, rw_dir): assert sm_mod.head.ref.name == sm_fb.name assert sm_mod.commit() == sm_fb.commit - # Create new branch which is in our past, and thus seemingly unrelated to the currently checked out one. + # Create new branch which is in our past, and thus seemingly unrelated to the + # currently checked out one. # To make it even 'harder', we shall fork and create a new commit. sm_pfb = sm_source_repo.create_head("past-feature", commit="HEAD~20") sm_pfb.checkout() @@ -1043,8 +1060,8 @@ def test_branch_renames(self, rw_dir): # Test submodule updates - must fail if submodule is dirty. touch(osp.join(sm_mod.working_tree_dir, "unstaged file")) - # This doesn't fail as our own submodule binsha didn't change, and the reset is only triggered if - # to_latest_revision is True. + # This doesn't fail as our own submodule binsha didn't change, and the reset is + # only triggered if to_latest_revision is True. parent_repo.submodule_update(to_latest_revision=False) assert sm_mod.head.ref.name == sm_pfb.name, "should have been switched to past head" assert sm_mod.commit() == sm_fb.commit, "Head wasn't reset" @@ -1184,8 +1201,8 @@ def test_submodule_add_unsafe_url_allowed(self, rw_repo): "fd::/foo", ] for url in urls: - # The URL will be allowed into the command, but the command will - # fail since we don't have that protocol enabled in the Git config file. + # The URL will be allowed into the command, but the command will fail + # since we don't have that protocol enabled in the Git config file. with self.assertRaises(GitCommandError): Submodule.add(rw_repo, "new", "new", url, allow_unsafe_protocols=True) assert not tmp_file.exists() @@ -1269,8 +1286,8 @@ def test_submodule_update_unsafe_url_allowed(self, rw_repo): ] for url in urls: submodule = Submodule(rw_repo, b"\0" * 20, name="new", path="new", url=url) - # The URL will be allowed into the command, but the command will - # fail since we don't have that protocol enabled in the Git config file. + # The URL will be allowed into the command, but the command will fail + # since we don't have that protocol enabled in the Git config file. with self.assertRaises(GitCommandError): submodule.update(allow_unsafe_protocols=True) assert not tmp_file.exists() diff --git a/test/test_util.py b/test/test_util.py index 65f77082d..824b3ab3d 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -153,7 +153,8 @@ def _patch_for_wrapping_test(self, mocker, hide_windows_known_errors): reason="PermissionError is only ever wrapped on Windows", ) def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): - """rmtree wraps PermissionError on Windows when HIDE_WINDOWS_KNOWN_ERRORS is true.""" + """rmtree wraps PermissionError on Windows when HIDE_WINDOWS_KNOWN_ERRORS is + true.""" self._patch_for_wrapping_test(mocker, True) with pytest.raises(SkipTest): @@ -171,7 +172,8 @@ def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): ], ) def test_does_not_wrap_perm_error_unless_enabled(self, mocker, permission_error_tmpdir, hide_windows_known_errors): - """rmtree does not wrap PermissionError on non-Windows systems or when HIDE_WINDOWS_KNOWN_ERRORS is false.""" + """rmtree does not wrap PermissionError on non-Windows systems or when + HIDE_WINDOWS_KNOWN_ERRORS is false.""" self._patch_for_wrapping_test(mocker, hide_windows_known_errors) with pytest.raises(PermissionError): @@ -182,7 +184,9 @@ def test_does_not_wrap_perm_error_unless_enabled(self, mocker, permission_error_ @pytest.mark.parametrize("hide_windows_known_errors", [False, True]) def test_does_not_wrap_other_errors(self, tmp_path, mocker, hide_windows_known_errors): - file_not_found_tmpdir = tmp_path / "testdir" # It is deliberately never created. + # The file is deliberately never created. + file_not_found_tmpdir = tmp_path / "testdir" + self._patch_for_wrapping_test(mocker, hide_windows_known_errors) with pytest.raises(FileNotFoundError): @@ -502,7 +506,8 @@ def test_actor_get_uid_laziness_called(self, mock_get_uid): committer = Actor.committer(None) author = Actor.author(None) # We can't test with `self.rorepo.config_reader()` here, as the UUID laziness - # depends on whether the user running the test has their global user.name config set. + # depends on whether the user running the test has their global user.name config + # set. self.assertEqual(committer.name, "user") self.assertTrue(committer.email.startswith("user@")) self.assertEqual(author.name, "user") From 4b04d8a33371d655a3cb5416ba18215fa08e0edd Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 21:34:30 -0500 Subject: [PATCH 0712/1392] Better clarify Submodule.branch_path documentation This revisits the changes in 3813bfb and takes a different approach, using the phrase "full repository-relative path" that is used elsewhere in the documentation and seems clear in context. Although this brings back the potential confusion around having the terms "full" and "relative" used together to describe a path, this may no longer be an issue now that the phrase "repository-relative" is used here as it is elsewhere. (In particular, see the SymbolicReference.to_full_path docstring.) --- git/objects/submodule/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index dc1deee36..6379d1500 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -134,7 +134,7 @@ def __init__( The URL to the remote repository which is the submodule. :param branch_path: - Complete relative path to ref to checkout when cloning the remote + Full repository-relative path to ref to checkout when cloning the remote repository. """ super().__init__(repo, binsha, mode, path) @@ -1473,8 +1473,8 @@ def branch(self) -> "Head": def branch_path(self) -> PathLike: """ :return: - Complete relative path as string to the branch we would checkout from the - remote and track + Full repository-relative path as string to the branch we would checkout from + the remote and track """ return self._branch_path From 254c82a00a3a6d141cfb6df7fdb8f33a23d4ebcb Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 21:58:22 -0500 Subject: [PATCH 0713/1392] More docstring revisions within git.refs --- git/refs/head.py | 32 +++++++++++++++---------------- git/refs/log.py | 8 ++++---- git/refs/reference.py | 7 ++++--- git/refs/remote.py | 2 +- git/refs/symbolic.py | 44 +++++++++++++++++++++---------------------- 5 files changed, 47 insertions(+), 46 deletions(-) diff --git a/git/refs/head.py b/git/refs/head.py index a051748ba..d546cb4ef 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -36,8 +36,8 @@ def strip_quotes(string: str) -> str: class HEAD(SymbolicReference): - """Special case of a SymbolicReference representing the repository's - HEAD reference.""" + """Special case of a SymbolicReference representing the repository's HEAD + reference.""" _HEAD_NAME = "HEAD" _ORIG_HEAD_NAME = "ORIG_HEAD" @@ -66,22 +66,21 @@ def reset( paths: Union[PathLike, Sequence[PathLike], None] = None, **kwargs: Any, ) -> "HEAD": - """Reset our HEAD to the given commit optionally synchronizing - the index and working tree. The reference we refer to will be set to commit as - well. + """Reset our HEAD to the given commit optionally synchronizing the index and + working tree. The reference we refer to will be set to commit as well. :param commit: :class:`~git.objects.commit.Commit`, :class:`~git.refs.reference.Reference`, or string identifying a revision we should reset HEAD to. :param index: - If True, the index will be set to match the given commit. + If ``True``, the index will be set to match the given commit. Otherwise it will not be touched. :param working_tree: - If True, the working tree will be forcefully adjusted to match the given + If ``True``, the working tree will be forcefully adjusted to match the given commit, possibly overwriting uncommitted changes without warning. - If `working_tree` is True, `index` must be True as well. + If `working_tree` is ``True``, `index` must be ``True`` as well. :param paths: Single path or list of paths relative to the git root directory @@ -90,7 +89,8 @@ def reset( :param kwargs: Additional arguments passed to ``git reset``. - :return: self + :return: + self """ mode: Union[str, None] mode = "--soft" @@ -151,8 +151,8 @@ def delete(cls, repo: "Repo", *heads: "Union[Head, str]", force: bool = False, * """Delete the given heads. :param force: - If True, the heads will be deleted even if they are not yet merged into the - main development stream. Default False. + If ``True``, the heads will be deleted even if they are not yet merged into + the main development stream. Default ``False``. """ flag = "-d" if force: @@ -193,7 +193,7 @@ def set_tracking_branch(self, remote_reference: Union["RemoteReference", None]) def tracking_branch(self) -> Union["RemoteReference", None]: """ :return: - The remote reference we are tracking, or None if we are not a tracking + The remote reference we are tracking, or ``None`` if we are not a tracking branch. """ from .remote import RemoteReference @@ -219,14 +219,14 @@ def rename(self, new_path: PathLike, force: bool = False) -> "Head": The prefix ``refs/heads`` is implied. :param force: - If True, the rename will succeed even if a head with the target name + If ``True``, the rename will succeed even if a head with the target name already exists. :return: self :note: - Respects the ref log as git commands are used. + Respects the ref log, as git commands are used. """ flag = "-m" if force: @@ -244,8 +244,8 @@ def checkout(self, force: bool = False, **kwargs: Any) -> Union["HEAD", "Head"]: The command will fail if changed working tree files would be overwritten. :param force: - If True, changes to the index and the working tree will be discarded. - If False, :class:`~git.exc.GitCommandError` will be raised in that + If ``True``, changes to the index and the working tree will be discarded. + If ``False``, :class:`~git.exc.GitCommandError` will be raised in that situation. :param kwargs: diff --git a/git/refs/log.py b/git/refs/log.py index 260f2fff5..7aefeb4e6 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -104,7 +104,7 @@ def new( tz_offset: int, message: str, ) -> "RefLogEntry": # skipcq: PYL-W0621 - """:return: New instance of a RefLogEntry""" + """:return: New instance of a :class:`RefLogEntry`""" if not isinstance(actor, Actor): raise ValueError("Need actor instance, got %s" % actor) # END check types @@ -112,7 +112,7 @@ def new( @classmethod def from_line(cls, line: bytes) -> "RefLogEntry": - """:return: New RefLogEntry instance from the given revlog line. + """:return: New :class:`RefLogEntry` instance from the given revlog line. :param line: Line bytes without trailing newline @@ -311,7 +311,7 @@ def append_entry( :param config_reader: Configuration reader of the repository - used to obtain user information. May also be an :class:`~git.util.Actor` instance identifying the committer - directly or None. + directly or ``None``. :param filepath: Full path to the log file. @@ -326,7 +326,7 @@ def append_entry( Message describing the change to the reference. :param write: - If True, the changes will be written right away. + If ``True``, the changes will be written right away. Otherwise the change will not be written. :return: diff --git a/git/refs/reference.py b/git/refs/reference.py index 32547278f..d1a9f3c54 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -65,7 +65,7 @@ def __init__(self, repo: "Repo", path: PathLike, check_path: bool = True) -> Non e.g. ``refs/heads/master``. :param check_path: - If False, you can provide any path. + If ``False``, you can provide any path. Otherwise the path must start with the default path prefix of this type. """ if check_path and not str(path).startswith(self._common_path_default + "/"): @@ -141,8 +141,9 @@ def iter_items( *args: Any, **kwargs: Any, ) -> Iterator[T_References]: - """Equivalent to SymbolicReference.iter_items, but will return non-detached - references as well.""" + """Equivalent to + :meth:`SymbolicReference.iter_items `, + but will return non-detached references as well.""" return cls._iter_items(repo, common_path) # }END interface diff --git a/git/refs/remote.py b/git/refs/remote.py index c2c2c1aac..bb2a4e438 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -56,7 +56,7 @@ def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: """Delete the given remote references. :note: - kwargs are given for comparability with the base class method as we + `kwargs` are given for comparability with the base class method as we should not narrow the signature. """ repo.git.branch("-d", "-r", *refs) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 6b9fd9ab7..70af4615d 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -63,7 +63,7 @@ class SymbolicReference: This does not point to a specific commit, but to another :class:`~git.refs.head.Head`, which itself specifies a commit. - A typical example for a symbolic reference is ``HEAD``. + A typical example for a symbolic reference is :class:`~git.refs.head.HEAD`. """ __slots__ = ("repo", "path") @@ -115,8 +115,8 @@ def _get_packed_refs_path(cls, repo: "Repo") -> str: @classmethod def _iter_packed_refs(cls, repo: "Repo") -> Iterator[Tuple[str, str]]: - """Return an iterator yielding pairs of sha1/path pairs (as strings) - for the corresponding refs. + """Return an iterator yielding pairs of sha1/path pairs (as strings) for the + corresponding refs. :note: The packed refs file will be kept open as long as we iterate. @@ -226,9 +226,9 @@ def _get_ref_info_helper( """ :return: (str(sha), str(target_ref_path)) if available, the sha the file at rela_path - points to, or None. + points to, or ``None``. - target_ref_path is the reference we point to, or None. + target_ref_path is the reference we point to, or ``None``. """ if ref_path: cls._check_ref_name_valid(ref_path) @@ -273,7 +273,7 @@ def _get_ref_info(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> Union[T :return: (str(sha), str(target_ref_path)) if available, the sha the file at rela_path points to, or None. - target_ref_path is the reference we point to, or None. + target_ref_path is the reference we point to, or ``None``. """ return cls._get_ref_info_helper(repo, ref_path) @@ -313,7 +313,7 @@ def set_commit( :class:`~git.objects.commit.Commit`. :raise ValueError: - If `commit` is not a :class:`~git.objects.commit.Commit` object or doesn't + If `commit` is not a :class:`~git.objects.commit.Commit` object, nor does it point to a commit. :return: @@ -357,7 +357,7 @@ def set_object( to. :param logmsg: - If not None, the message will be used in the reflog entry to be written. + If not ``None``, the message will be used in the reflog entry to be written. Otherwise the reflog is not altered. :note: @@ -491,8 +491,8 @@ def set_reference( def is_valid(self) -> bool: """ :return: - True if the reference is valid, hence it can be read and points to a valid - object or reference. + ``True`` if the reference is valid, hence it can be read and points to a + valid object or reference. """ try: self.object @@ -505,7 +505,7 @@ def is_valid(self) -> bool: def is_detached(self) -> bool: """ :return: - True if we are a detached reference, hence we point to a specific commit + ``True`` if we are a detached reference, hence we point to a specific commit instead to another reference. """ try: @@ -565,7 +565,7 @@ def log_append( def log_entry(self, index: int) -> "RefLogEntry": """ :return: - RefLogEntry at the given index + :class:`~git.refs.log.RefLogEntry` at the given index :param index: Python list compatible positive or negative index. @@ -582,7 +582,7 @@ def to_full_path(cls, path: Union[PathLike, "SymbolicReference"]) -> PathLike: """ :return: String with a full repository-relative path which can be used to initialize - a Reference instance, for instance by using + a :class:`~git.refs.reference.Reference` instance, for instance by using :meth:`Reference.from_path `. """ if isinstance(path, SymbolicReference): @@ -666,7 +666,7 @@ def _create( ) -> T_References: """Internal method used to create a new symbolic reference. - If `resolve` is False, the reference will be taken as is, creating a proper + If `resolve` is ``False``, the reference will be taken as is, creating a proper symbolic reference. Otherwise it will be resolved to the corresponding object and a detached symbolic reference will be created instead. """ @@ -722,12 +722,12 @@ def create( If it is a commit-ish, the symbolic ref will be detached. :param force: - If True, force creation even if a symbolic reference with that name already - exists. Raise :class:`OSError` otherwise. + If ``True``, force creation even if a symbolic reference with that name + already exists. Raise :class:`OSError` otherwise. :param logmsg: - If not None, the message to append to the reflog. - If None, no reflog entry is written. + If not ``None``, the message to append to the reflog. + If ``None``, no reflog entry is written. :return: Newly created symbolic reference @@ -751,8 +751,8 @@ def rename(self, new_path: PathLike, force: bool = False) -> "SymbolicReference" In case this is a symbolic ref, there is no implied prefix. :param force: - If True, the rename will succeed even if a head with the target name already - exists. It will be overwritten in that case. + If ``True``, the rename will succeed even if a head with the target name + already exists. It will be overwritten in that case. :return: self @@ -847,8 +847,8 @@ def iter_items( :param common_path: Optional keyword argument to the path which is to be shared by all returned Ref objects. - Defaults to class specific portion if None, ensuring that only refs suitable - for the actual class are returned. + Defaults to class specific portion if ``None``, ensuring that only refs + suitable for the actual class are returned. :return: A list of :class:`SymbolicReference`, each guaranteed to be a symbolic ref From 679d2e87b4a4c758a1059854f03fb6b4b2fd28d3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 22:01:26 -0500 Subject: [PATCH 0714/1392] Fix exception type in require_remote_ref_path docstring The require_remote_ref_path decorator has always used ValueError (ever since its introduction in a92ab80) but was documented as using TypeError. ValueError is the correct exception to raise here, since this is not any kind of type error or related condition. So the bug wasn't in the function's behavior, but instead in the way that behavior was documented. (The bug was fairly minor, since the function is not listed in __all__.) --- git/refs/reference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/refs/reference.py b/git/refs/reference.py index d1a9f3c54..a7b545fed 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -25,7 +25,7 @@ def require_remote_ref_path(func: Callable[..., _T]) -> Callable[..., _T]: - """A decorator raising :class:`TypeError` if we are not a valid remote, based on the + """A decorator raising :class:`ValueError` if we are not a valid remote, based on the path.""" def wrapper(self: T_References, *args: Any) -> _T: From ee0301ab69e4f2af1248596a9efb897893c54da1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 23:13:54 -0500 Subject: [PATCH 0715/1392] More docstring revisions in second-level modules and git.__init__ --- git/__init__.py | 2 +- git/cmd.py | 36 +++++++++++++++++++----------------- git/config.py | 11 ++++------- git/db.py | 3 ++- git/diff.py | 17 +++++++++-------- git/remote.py | 39 +++++++++++++++++++++------------------ git/util.py | 6 +++--- 7 files changed, 59 insertions(+), 55 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index 6fc6110f4..ed8a88d4b 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -127,7 +127,7 @@ def refresh(path: Optional[PathLike] = None) -> None: immediately, relative to the current directory. :note: - The *path* parameter is usually omitted and cannot be used to specify a custom + The `path` parameter is usually omitted and cannot be used to specify a custom command whose location is looked up in a path search on each call. See :meth:`Git.refresh ` for details on how to achieve this. diff --git a/git/cmd.py b/git/cmd.py index d0c76eafe..949814765 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -106,7 +106,7 @@ def handle_process_output( decode_streams: bool = True, kill_after_timeout: Union[None, float] = None, ) -> None: - """Register for notifications to learn that process output is ready to read, and + R"""Register for notifications to learn that process output is ready to read, and dispatch lines to the respective line handlers. This function returns once the finalizer returns. @@ -126,8 +126,11 @@ def handle_process_output( :param decode_streams: Assume stdout/stderr streams are binary and decode them before pushing their contents to handlers. - Set this to ``False`` if ``universal_newlines == True`` (then streams are in - text mode) or if decoding must happen later (i.e. for :class:`~git.diff.Diff`s). + + This defaults to ``True``. Set it to ``False``: + + - if ``universal_newlines == True``, as then streams are in text mode, or + - if decoding must happen later, such as for :class:`~git.diff.Diff`\s. :param kill_after_timeout: :class:`float` or ``None``, Default = ``None`` @@ -379,15 +382,14 @@ def __setstate__(self, d: Dict[str, Any]) -> None: :note: The git executable is actually found during the refresh step in the top level - :mod:`__init__`. It can also be changed by explicitly calling - :func:`git.refresh`. + ``__init__``. It can also be changed by explicitly calling :func:`git.refresh`. """ _refresh_token = object() # Since None would match an initial _version_info_token. @classmethod def refresh(cls, path: Union[None, PathLike] = None) -> bool: - """This gets called by the refresh function (see the top level __init__). + """This gets called by the refresh function (see the top level ``__init__``). :param path: Optional path to the git executable. If not absolute, it is resolved @@ -868,8 +870,7 @@ def __init__(self, working_dir: Union[None, PathLike] = None): self.cat_file_all: Union[None, TBD] = None def __getattr__(self, name: str) -> Any: - """A convenience method as it allows to call the command as if it was - an object. + """A convenience method as it allows to call the command as if it was an object. :return: Callable object that will execute call :meth:`_call_process` with your @@ -899,7 +900,7 @@ def working_dir(self) -> Union[None, PathLike]: @property def version_info(self) -> Tuple[int, ...]: """ - :return: tuple with integers representing the major, minor and additional + :return: Tuple with integers representing the major, minor and additional version numbers as parsed from ``git version``. Up to four fields are used. This value is generated on demand and is cached. @@ -1021,7 +1022,7 @@ def execute( :param output_stream: If set to a file-like object, data produced by the git command will be copied to the given stream instead of being returned as a string. - This feature only has any effect if `as_process` is False. + This feature only has any effect if `as_process` is ``False``. :param stdout_as_string: If ``False``, the command's standard output will be bytes. Otherwise, it @@ -1030,10 +1031,10 @@ def execute( :param kill_after_timeout: Specifies a timeout in seconds for the git command, after which the process - should be killed. This will have no effect if `as_process` is set to True. - It is set to None by default and will let the process run until the timeout - is explicitly specified. Uses of this feature should be carefully - considered, due to the following limitations: + should be killed. This will have no effect if `as_process` is set to + ``True``. It is set to ``None`` by default and will let the process run + until the timeout is explicitly specified. Uses of this feature should be + carefully considered, due to the following limitations: 1. This feature is not supported at all on Windows. 2. Effectiveness may vary by operating system. ``ps --ppid`` is used to @@ -1099,7 +1100,7 @@ def execute( :note: If you add additional keyword arguments to the signature of this method, - you must update the execute_kwargs tuple housed in this module. + you must update the ``execute_kwargs`` variable housed in this module. """ # Remove password for the command if present. redacted_command = remove_password_if_present(command) @@ -1420,9 +1421,10 @@ def _call_process( :param kwargs: Contains key-values for the following: - - The :meth:`execute()` kwds, as listed in :var:`execute_kwargs`. + - The :meth:`execute()` kwds, as listed in ``execute_kwargs``. - "Command options" to be converted by :meth:`transform_kwargs`. - - The ``insert_kwargs_after`` key which its value must match one of ``*args``. + - The ``insert_kwargs_after`` key which its value must match one of + ``*args``. It also contains any command options, to be appended after the matched arg. diff --git a/git/config.py b/git/config.py index ff5c9d564..b358ee417 100644 --- a/git/config.py +++ b/git/config.py @@ -411,7 +411,7 @@ def release(self) -> None: not be used anymore afterwards. In Python 3, it's required to explicitly release locks and flush changes, as - :meth:`__del__` is not called deterministically anymore. + ``__del__`` is not called deterministically anymore. """ # Checking for the lock here makes sure we do not raise during write() # in case an invalid parser was created who could not get a lock. @@ -539,7 +539,7 @@ def _included_paths(self) -> List[Tuple[str, str]]: """List all paths that must be included to configuration. :return: - The list of paths, where each path is a tuple of ``(option, value)``. + The list of paths, where each path is a tuple of (option, value). """ paths = [] @@ -591,9 +591,6 @@ def read(self) -> None: # type: ignore[override] This will ignore files that cannot be read, possibly leaving an empty configuration. - :return: - Nothing - :raise IOError: If a file cannot be handled. """ @@ -765,7 +762,7 @@ def add_section(self, section: str) -> None: @property def read_only(self) -> bool: - """:return: True if this instance may change the configuration file""" + """:return: ``True`` if this instance may change the configuration file""" return self._read_only # FIXME: Figure out if default or return type can really include bool. @@ -918,7 +915,7 @@ def add_value(self, section: str, option: str, value: Union[str, bytes, int, flo return self def rename_section(self, section: str, new_name: str) -> "GitConfigParser": - """Rename the given section to new_name. + """Rename the given section to `new_name`. :raise ValueError: If: diff --git a/git/db.py b/git/db.py index eb7f758da..3f496a979 100644 --- a/git/db.py +++ b/git/db.py @@ -54,7 +54,8 @@ def stream(self, binsha: bytes) -> OStream: def partial_to_complete_sha_hex(self, partial_hexsha: str) -> bytes: """ - :return: Full binary 20 byte sha from the given partial hexsha + :return: + Full binary 20 byte sha from the given partial hexsha :raise gitdb.exc.AmbiguousObjectName: diff --git a/git/diff.py b/git/diff.py index 3c760651b..adddaa7f6 100644 --- a/git/diff.py +++ b/git/diff.py @@ -121,9 +121,9 @@ def diff( * If ``None``, we will be compared to the working tree. * If :class:`~git.index.base.Treeish`, it will be compared against the respective tree. - * If :class:`~Diffable.Index`, it will be compared against the index. + * If :class:`Diffable.Index`, it will be compared against the index. * If :attr:`git.NULL_TREE`, it will compare against the empty tree. - * It defaults to :class:`~Diffable.Index` so that the method will not by + * It defaults to :class:`Diffable.Index` so that the method will not by default fail on bare repositories. :param paths: @@ -280,11 +280,11 @@ class Diff: Working Tree Blobs: When comparing to working trees, the working tree blob will have a null hexsha - as a corresponding object does not yet exist. The mode will be null as well. - The path will be available, though. + as a corresponding object does not yet exist. The mode will be null as well. The + path will be available, though. - If it is listed in a diff, the working tree version of the file must - differ from the version in the index or tree, and hence has been modified. + If it is listed in a diff, the working tree version of the file must differ from + the version in the index or tree, and hence has been modified. """ # Precompiled regex. @@ -468,7 +468,8 @@ def rename_to(self) -> Optional[str]: @property def renamed(self) -> bool: - """ + """Deprecated, use :attr:`renamed_file` instead. + :return: ``True`` if the blob of our diff has been renamed @@ -480,7 +481,7 @@ def renamed(self) -> bool: @property def renamed_file(self) -> bool: - """:return: True if the blob of our diff has been renamed""" + """:return: ``True`` if the blob of our diff has been renamed""" return self.rename_from != self.rename_to @classmethod diff --git a/git/remote.py b/git/remote.py index 3d5b8fdc4..6ce720ee3 100644 --- a/git/remote.py +++ b/git/remote.py @@ -338,7 +338,7 @@ class FetchInfo(IterableObj): @classmethod def refresh(cls) -> Literal[True]: - """This gets called by the refresh function (see the top level __init__).""" + """This gets called by the refresh function (see the top level ``__init__``).""" # Clear the old values in _flag_map. with contextlib.suppress(KeyError): del cls._flag_map["t"] @@ -386,19 +386,22 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": """Parse information from the given line as returned by ``git-fetch -v`` and return a new :class:`FetchInfo` object representing this information. - We can handle a line as follows: - "%c %-\\*s %-\\*s -> %s%s" + We can handle a line as follows:: - Where c is either ' ', !, +, -, \\*, or = - ! means error - + means success forcing update - - means a tag was updated - * means birth of new branch or tag - = means the head was up to date (and not moved) - ' ' means a fast-forward + %c %-*s %-*s -> %s%s - fetch line is the corresponding line from FETCH_HEAD, like - acb0fa8b94ef421ad60c8507b634759a472cd56c not-for-merge branch '0.1.7RC' of /tmp/tmpya0vairemote_repo + Where ``c`` is either a space, ``!``, ``+``, ``-``, ``*``, or ``=``: + + - '!' means error + - '+' means success forcing update + - '-' means a tag was updated + - '*' means birth of new branch or tag + - '=' means the head was up to date (and not moved) + - ' ' means a fast-forward + + `fetch_line` is the corresponding line from FETCH_HEAD, like:: + + acb0fa8b94ef421ad60c8507b634759a472cd56c not-for-merge branch '0.1.7RC' of /tmp/tmpya0vairemote_repo """ match = cls._re_fetch_result.match(line) if match is None: @@ -625,7 +628,7 @@ def exists(self) -> bool: @classmethod def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator["Remote"]: - """:return: Iterator yielding Remote objects of the given repository""" + """:return: Iterator yielding :class:`Remote` objects of the given repository""" for section in repo.config_reader("repository").sections(): if not section.startswith("remote "): continue @@ -639,7 +642,7 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator["Remote def set_url( self, new_url: str, old_url: Optional[str] = None, allow_unsafe_protocols: bool = False, **kwargs: Any ) -> "Remote": - """Configure URLs on current remote (cf command ``git remote set-url``). + """Configure URLs on current remote (cf. command ``git remote set-url``). This command manages URLs on the remote. @@ -1020,7 +1023,7 @@ def fetch( facility. :param progress: - See :meth:`push` method. + See the :meth:`push` method. :param verbose: Boolean for verbose output. @@ -1081,8 +1084,8 @@ def pull( allow_unsafe_options: bool = False, **kwargs: Any, ) -> IterableList[FetchInfo]: - """Pull changes from the given branch, being the same as a fetch followed - by a merge of branch with your local branch. + """Pull changes from the given branch, being the same as a fetch followed by a + merge of branch with your local branch. :param refspec: See :meth:`fetch` method. @@ -1157,7 +1160,7 @@ def push( :param kill_after_timeout: To specify a timeout in seconds for the git command, after which the process - should be killed. It is set to None by default. + should be killed. It is set to ``None`` by default. :param allow_unsafe_protocols: Allow unsafe protocols to be used, like ``ext``. diff --git a/git/util.py b/git/util.py index e7dd94a08..bab765a09 100644 --- a/git/util.py +++ b/git/util.py @@ -762,9 +762,9 @@ def update(self, *args: Any, **kwargs: Any) -> None: class Actor: - """Actors hold information about a person acting on the repository. They - can be committers and authors or anything with a name and an email as mentioned in - the git log entries.""" + """Actors hold information about a person acting on the repository. They can be + committers and authors or anything with a name and an email as mentioned in the git + log entries.""" # PRECOMPILED REGEX name_only_regex = re.compile(r"<(.*)>") From 231c3ef758a004bb51fae2868cb59742069c7dde Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 23:23:45 -0500 Subject: [PATCH 0716/1392] More docstring revisions within git.repo --- git/repo/base.py | 15 +++++++-------- git/repo/fun.py | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index 4745a2411..a54591746 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -102,9 +102,8 @@ class BlameEntry(NamedTuple): class Repo: - """Represents a git repository and allows you to query references, father - commit information, generate diffs, create and clone repositories, and query the - log. + """Represents a git repository and allows you to query references, create commit + information, generate diffs, create and clone repositories, and query the log. The following attributes are worth using: @@ -112,8 +111,8 @@ class Repo: working tree directory if available or the ``.git`` directory in case of bare repositories. - * :attr:`working_tree_dir` is the working tree directory, but will return None if we - are a bare repository. + * :attr:`working_tree_dir` is the working tree directory, but will return ``None`` + if we are a bare repository. * :attr:`git_dir` is the ``.git`` repository directory, which is always set. """ @@ -596,7 +595,7 @@ def create_tag( return TagReference.create(self, path, ref, message, force, **kwargs) def delete_tag(self, *tags: TagReference) -> None: - """Delete the given tag references""" + """Delete the given tag references.""" return TagReference.delete(self, *tags) def create_remote(self, name: str, url: str, **kwargs: Any) -> Remote: @@ -775,7 +774,7 @@ def iter_commits( def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: R"""Find the closest common ancestor for the given revision (:class:`~git.objects.commit.Commit`\s, :class:`~git.refs.tag.Tag`\s, - :class:`git.refs.reference.Reference`\s, etc.). + :class:`~git.refs.reference.Reference`\s, etc.). :param rev: At least two revs to find the common ancestor for. @@ -1030,7 +1029,7 @@ def active_branch(self) -> Head: If HEAD is detached. :return: - Head to the active branch + :class:`~git.refs.head.Head` to the active branch """ # reveal_type(self.head.reference) # => Reference return self.head.reference diff --git a/git/repo/fun.py b/git/repo/fun.py index e9fad2c46..e3c69c68c 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -151,7 +151,7 @@ def name_to_object( :param return_ref: If ``True``, and name specifies a reference, we will return the reference - instead of the object. Otherwise it will raise `~gitdb.exc.BadObject` o + instead of the object. Otherwise it will raise `~gitdb.exc.BadObject` or `~gitdb.exc.BadName`. """ hexsha: Union[None, str, bytes] = None From e166a0a7b9dc2643c686e0a5b365e22d0fa88ae7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 23:45:33 -0500 Subject: [PATCH 0717/1392] More docstring revisions within git.objects --- git/objects/base.py | 2 +- git/objects/commit.py | 15 ++++++++------- git/objects/fun.py | 4 ++-- git/objects/tag.py | 2 +- git/objects/tree.py | 10 +++++----- git/objects/util.py | 15 ++++++++------- 6 files changed, 25 insertions(+), 23 deletions(-) diff --git a/git/objects/base.py b/git/objects/base.py index 5cee8e405..e78420e8a 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -98,7 +98,7 @@ def new_from_sha(cls, repo: "Repo", sha1: bytes) -> Commit_ish: New object instance of a type appropriate to represent the given binary sha1 :param sha1: - 20 byte binary sha1 + 20 byte binary sha1. """ if sha1 == cls.NULL_BIN_SHA: # The NULL binsha is always the root commit. diff --git a/git/objects/commit.py b/git/objects/commit.py index b5b1c540d..06ab0898b 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -298,7 +298,7 @@ def iter_items( The :class:`~git.repo.base.Repo`. :param rev: - Revision specifier, see git-rev-parse for viable options. + Revision specifier. See git-rev-parse for viable options. :param paths: An optional path or list of paths. If set only :class:`Commit`\s that @@ -309,7 +309,7 @@ def iter_items( * ``max_count`` is the maximum number of commits to fetch. * ``skip`` is the number of commits to skip. - * ``since`` all commits since e.g. '1970-01-01'. + * ``since`` selects all commits since some date, e.g. ``"1970-01-01"``. :return: Iterator yielding :class:`Commit` items. @@ -380,12 +380,13 @@ def stats(self) -> Stats: def trailers(self) -> Dict[str, str]: """Deprecated. Get the trailers of the message as a dictionary. - :note: This property is deprecated, please use either :attr:`trailers_list` or - :attr:`trailers_dict``. + :note: + This property is deprecated, please use either :attr:`trailers_list` or + :attr:`trailers_dict`. :return: - Dictionary containing whitespace stripped trailer information. Only contains - the latest instance of each trailer key. + Dictionary containing whitespace stripped trailer information. + Only contains the latest instance of each trailer key. """ return {k: v[0] for k, v in self.trailers_dict.items()} @@ -539,7 +540,7 @@ def create_from_tree( author_date: Union[None, str, datetime.datetime] = None, commit_date: Union[None, str, datetime.datetime] = None, ) -> "Commit": - """Commit the given tree, creating a commit object. + """Commit the given tree, creating a :class:`Commit` object. :param repo: :class:`~git.repo.base.Repo` object the commit should be part of. diff --git a/git/objects/fun.py b/git/objects/fun.py index ad5bc9a88..22b99cb6b 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -186,8 +186,8 @@ def traverse_trees_recursive( given tree. :param tree_shas: - Iterable of shas pointing to trees. All trees must be on the same level. A - tree-sha may be ``None`` in which case ``None``. + Iterable of shas pointing to trees. All trees must be on the same level. + A tree-sha may be ``None``, in which case ``None``. :param path_prefix: A prefix to be added to the returned paths on this level. diff --git a/git/objects/tag.py b/git/objects/tag.py index 7589a5be1..f455c55fc 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -3,7 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Provides an :class:`git.objects.base.Object`-based type for annotated tags. +"""Provides an :class:`~git.objects.base.Object`-based type for annotated tags. This defines the :class:`TagReference` class, which represents annotated tags. For lightweight tags, see the :mod:`git.refs.tag` module. diff --git a/git/objects/tree.py b/git/objects/tree.py index 69cd6ef3f..a506bba7d 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -166,7 +166,7 @@ class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): Tree as a list: - * Access a specific blob using the ``tree['filename']`` notation. + * Access a specific blob using the ``tree["filename"]`` notation. * You may likewise access by index, like ``blob = tree[0]``. """ @@ -215,8 +215,8 @@ def _set_cache_(self, attr: str) -> None: # END handle attribute def _iter_convert_to_object(self, iterable: Iterable[TreeCacheTup]) -> Iterator[IndexObjUnion]: - """Iterable yields tuples of (binsha, mode, name), which will be converted - to the respective object representation. + """Iterable yields tuples of (binsha, mode, name), which will be converted to + the respective object representation. """ for binsha, mode, name in iterable: path = join_path(self.path, name) @@ -338,8 +338,8 @@ def traverse( def list_traverse(self, *args: Any, **kwargs: Any) -> IterableList[IndexObjUnion]: """ :return: - :class:`~git.util.IterableList`IterableList with the results of the - traversal as produced by :meth:`traverse` + :class:`~git.util.IterableList` with the results of the traversal as + produced by :meth:`traverse` Tree -> IterableList[Union[Submodule, Tree, Blob]] """ diff --git a/git/objects/util.py b/git/objects/util.py index 6f46bab18..6f4e7d087 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -474,7 +474,7 @@ def _traverse( ignore_self: int = 1, as_edge: bool = False, ) -> Union[Iterator[Union["Traversable", "Blob"]], Iterator[TraversedTup]]: - """Iterator yielding items found when traversing self. + """Iterator yielding items found when traversing `self`. :param predicate: A function ``f(i,d)`` that returns ``False`` if item i at depth ``d`` should @@ -485,9 +485,10 @@ def _traverse( item ``i`` at depth ``d``. Item ``i`` will not be returned. :param depth: - Defines at which level the iteration should not go deeper if -1, there is no - limit if 0, you would effectively only get self, the root of the iteration - i.e. if 1, you would only get the first level of predecessors/successors. + Defines at which level the iteration should not go deeper if -1. There is no + limit if 0, you would effectively only get `self`, the root of the + iteration. If 1, you would only get the first level of + predecessors/successors. :param branch_first: If ``True``, items will be returned branch first, otherwise depth first. @@ -497,8 +498,8 @@ def _traverse( encountered several times. Loops are prevented that way. :param ignore_self: - If ``True``, self will be ignored and automatically pruned from the result. - Otherwise it will be the first item to be returned. If `as_edge` is + If ``True``, `self` will be ignored and automatically pruned from the + result. Otherwise it will be the first item to be returned. If `as_edge` is ``True``, the source of the first edge is ``None``. :param as_edge: @@ -507,7 +508,7 @@ def _traverse( destination. :return: - Iterator yielding items found when traversing self:: + Iterator yielding items found when traversing `self`:: Commit -> Iterator[Union[Commit, Tuple[Commit, Commit]] Submodule -> Iterator[Submodule, Tuple[Submodule, Submodule]] Tree -> From ffeb7e76c54c3e8af1e94e116f3a1ef714b36e19 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 28 Feb 2024 23:57:35 -0500 Subject: [PATCH 0718/1392] More docstring revisions in git.objects.submodule.base --- git/objects/submodule/base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 6379d1500..cdd7c8e1b 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -172,7 +172,7 @@ def _set_cache_(self, attr: str) -> None: @classmethod def _get_intermediate_items(cls, item: "Submodule") -> IterableList["Submodule"]: - """:return: all the submodules of our module repository""" + """:return: All the submodules of our module repository""" try: return cls.list_items(item.module()) except InvalidGitRepositoryError: @@ -326,7 +326,7 @@ def _clone_repo( Allow unsafe options to be used, like ``--upload-pack``. :param kwargs: - Additional arguments given to ``git clone`` + Additional arguments given to ``git clone``. """ module_abspath = cls._module_abspath(repo, path, name) module_checkout_path = module_abspath @@ -351,7 +351,7 @@ def _clone_repo( @classmethod def _to_relative_path(cls, parent_repo: "Repo", path: PathLike) -> PathLike: - """:return: a path guaranteed to be relative to the given parent repository + """:return: A path guaranteed to be relative to the given parent repository :raise ValueError: If path is not contained in the parent repository's working tree. @@ -642,7 +642,7 @@ def update( If ``True``, the submodule's sha will be ignored during checkout. Instead, the remote will be fetched, and the local tracking branch updated. This only works if we have a local tracking branch, which is the case if the remote - repository had a master branch, or of the ``branch`` option was specified + repository had a master branch, or if the ``branch`` option was specified for this submodule and the branch existed remotely. :param progress: @@ -1309,7 +1309,7 @@ def config_writer( repository. :param write: - If True, the index will be written each time a configuration value changes. + If ``True``, the index will be written each time a configuration value changes. :note: The parameters allow for a more efficient writing of the index, as you can From ec9395526719a87127efa9e8b942a05a4238aa25 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 29 Feb 2024 00:18:48 -0500 Subject: [PATCH 0719/1392] Further refine some docstring revisions --- git/cmd.py | 16 +++++++++------- git/diff.py | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 949814765..6574cbb34 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -127,10 +127,10 @@ def handle_process_output( Assume stdout/stderr streams are binary and decode them before pushing their contents to handlers. - This defaults to ``True``. Set it to ``False``: + This defaults to ``True``. Set it to ``False`` if: - - if ``universal_newlines == True``, as then streams are in text mode, or - - if decoding must happen later, such as for :class:`~git.diff.Diff`\s. + - ``universal_newlines == True``, as then streams are in text mode, or + - decoding must happen later, such as for :class:`~git.diff.Diff`\s. :param kill_after_timeout: :class:`float` or ``None``, Default = ``None`` @@ -1085,13 +1085,15 @@ def execute( specify may not be the same ones. :return: - * str(output) if extended_output = False (Default) - * tuple(int(status), str(stdout), str(stderr)) if extended_output = True + * str(output), if `extended_output` is ``False`` (Default) + * tuple(int(status), str(stdout), str(stderr)), + if `extended_output` is ``True`` If `output_stream` is ``True``, the stdout value will be your output stream: - * output_stream if extended_output = False - * tuple(int(status), output_stream, str(stderr)) if extended_output = True + * output_stream, if `extended_output` is ``False`` + * tuple(int(status), output_stream, str(stderr)), + if `extended_output` is ``True`` Note that git is executed with ``LC_MESSAGES="C"`` to ensure consistent output regardless of system language. diff --git a/git/diff.py b/git/diff.py index adddaa7f6..a89ca9880 100644 --- a/git/diff.py +++ b/git/diff.py @@ -112,8 +112,8 @@ def diff( create_patch: bool = False, **kwargs: Any, ) -> "DiffIndex": - """Create diffs between two items being trees, trees and index or an - index and the working tree. Detects renames automatically. + """Create diffs between two items being trees, trees and index or an index and + the working tree. Detects renames automatically. :param other: This the item to compare us with. From 63983c2b9206a4a599112aa6c302a143f4661799 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 29 Feb 2024 00:41:10 -0500 Subject: [PATCH 0720/1392] Remove note in GitCmdObjectDB docstring For #1849. --- git/db.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/git/db.py b/git/db.py index 3f496a979..bf0de40de 100644 --- a/git/db.py +++ b/git/db.py @@ -30,10 +30,6 @@ class GitCmdObjectDB(LooseObjectDB): objects, pack files and an alternates file. It will create objects only in the loose object database. - - :note: - For now, we use the git command to do all the lookup, just until we have packs - and the other implementations. """ def __init__(self, root_path: PathLike, git: "Git") -> None: From f43292e4e4860c856de0ac52ef4a326aacff6579 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 29 Feb 2024 00:51:12 -0500 Subject: [PATCH 0721/1392] Somewhat improve _get_ref_info{,_helper} docstrings + Add a missing :class: reference in _get_commit docstring. --- git/refs/symbolic.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 70af4615d..142952e06 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -225,10 +225,10 @@ def _get_ref_info_helper( ) -> Union[Tuple[str, None], Tuple[None, str]]: """ :return: - (str(sha), str(target_ref_path)) if available, the sha the file at rela_path - points to, or ``None``. + *(str(sha), str(target_ref_path))*, where: - target_ref_path is the reference we point to, or ``None``. + * *sha* is of the file at rela_path points to if available, or ``None``. + * *target_ref_path* is the reference we point to, or ``None``. """ if ref_path: cls._check_ref_name_valid(ref_path) @@ -270,10 +270,11 @@ def _get_ref_info_helper( @classmethod def _get_ref_info(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> Union[Tuple[str, None], Tuple[None, str]]: """ - :return: (str(sha), str(target_ref_path)) if available, the sha the file at - rela_path points to, or None. + :return: + *(str(sha), str(target_ref_path))*, where: - target_ref_path is the reference we point to, or ``None``. + * *sha* is of the file at rela_path points to if available, or ``None``. + * *target_ref_path* is the reference we point to, or ``None``. """ return cls._get_ref_info_helper(repo, ref_path) @@ -290,9 +291,9 @@ def _get_object(self) -> Commit_ish: def _get_commit(self) -> "Commit": """ :return: - Commit object we point to. This works for detached and non-detached - :class:`SymbolicReference` instances. The symbolic reference will be - dereferenced recursively. + :class:`~git.objects.commit.Commit` object we point to. This works for + detached and non-detached :class:`SymbolicReference` instances. The symbolic + reference will be dereferenced recursively. """ obj = self._get_object() if obj.type == "tag": From 37c93de3d31aabf32f0032a2a49a9fee285fa6e8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 29 Feb 2024 03:44:44 -0500 Subject: [PATCH 0722/1392] A couple more small docstring refinements - Single vs. double backtick error/inconsistency in one place (parameter names were in in doubled backticks, now single) - Undo making "Iterator" in the phrase "Iterator yielding" a reference to collections.abc.Iterator, which is unnecessary and not done anywhere else. --- git/config.py | 4 ++-- git/util.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/git/config.py b/git/config.py index b358ee417..2164f67dc 100644 --- a/git/config.py +++ b/git/config.py @@ -920,8 +920,8 @@ def rename_section(self, section: str, new_name: str) -> "GitConfigParser": :raise ValueError: If: - * ``section`` doesn't exist. - * A section with ``new_name`` does already exist. + * `section` doesn't exist. + * A section with `new_name` does already exist. :return: This instance diff --git a/git/util.py b/git/util.py index bab765a09..52f9dbdad 100644 --- a/git/util.py +++ b/git/util.py @@ -1257,7 +1257,7 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> Iterator[T_Itera keyword arguments, subclasses are obliged to to yield all items. :return: - :class:`~collections.abc.Iterator` yielding Items + Iterator yielding Items """ raise NotImplementedError("To be implemented by Subclass") From 0d6c68af35d1d4a9da95b189db1da68fdcee40b8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 1 Mar 2024 14:04:54 -0500 Subject: [PATCH 0723/1392] Fix ref to git.refresh in refresh methods' docstrings --- git/cmd.py | 4 +++- git/remote.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 6574cbb34..6e71e37fd 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -389,7 +389,9 @@ def __setstate__(self, d: Dict[str, Any]) -> None: @classmethod def refresh(cls, path: Union[None, PathLike] = None) -> bool: - """This gets called by the refresh function (see the top level ``__init__``). + """This gets called by the :func:`git.refresh` function. + + See the top level ``__init__.py``. :param path: Optional path to the git executable. If not absolute, it is resolved diff --git a/git/remote.py b/git/remote.py index 6ce720ee3..c1b2c11f7 100644 --- a/git/remote.py +++ b/git/remote.py @@ -338,7 +338,10 @@ class FetchInfo(IterableObj): @classmethod def refresh(cls) -> Literal[True]: - """This gets called by the refresh function (see the top level ``__init__``).""" + """This gets called by the :func:`git.refresh` function. + + See the top level ``__init__.py``. + """ # Clear the old values in _flag_map. with contextlib.suppress(KeyError): del cls._flag_map["t"] From 65657421014ebcf89e29734ca2383346063c8363 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 1 Mar 2024 14:21:35 -0500 Subject: [PATCH 0724/1392] Further expand refresh methods' docstrings --- git/cmd.py | 7 +++++-- git/remote.py | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 6e71e37fd..4fd6dc7cd 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -389,9 +389,12 @@ def __setstate__(self, d: Dict[str, Any]) -> None: @classmethod def refresh(cls, path: Union[None, PathLike] = None) -> bool: - """This gets called by the :func:`git.refresh` function. + """Update information about the git executable :class:`Git` objects will use. - See the top level ``__init__.py``. + Called by the :func:`git.refresh` function in the top level ``__init__``. + + This gets called by the :func:`git.refresh` function in the top-level + ``__init__``. :param path: Optional path to the git executable. If not absolute, it is resolved diff --git a/git/remote.py b/git/remote.py index c1b2c11f7..2c671582b 100644 --- a/git/remote.py +++ b/git/remote.py @@ -338,9 +338,10 @@ class FetchInfo(IterableObj): @classmethod def refresh(cls) -> Literal[True]: - """This gets called by the :func:`git.refresh` function. + """Update information about which ``git fetch`` flags are supported by the git + executable being used. - See the top level ``__init__.py``. + Called by the :func:`git.refresh` function in the top level ``__init__``. """ # Clear the old values in _flag_map. with contextlib.suppress(KeyError): From c0a27c027d2e04e3379f68f9e3299f30730559c1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 1 Mar 2024 14:41:01 -0500 Subject: [PATCH 0725/1392] Better document overrides in GitCmdObjectDB --- git/db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/git/db.py b/git/db.py index bf0de40de..15e791e6c 100644 --- a/git/db.py +++ b/git/db.py @@ -38,11 +38,12 @@ def __init__(self, root_path: PathLike, git: "Git") -> None: self._git = git def info(self, binsha: bytes) -> OInfo: + """Get a git object header (using git itself).""" hexsha, typename, size = self._git.get_object_header(bin_to_hex(binsha)) return OInfo(hex_to_bin(hexsha), typename, size) def stream(self, binsha: bytes) -> OStream: - """For now, all lookup is done by git itself""" + """Get git object data as a stream supporting ``read()`` (using git itself).""" hexsha, typename, size, stream = self._git.stream_object_data(bin_to_hex(binsha)) return OStream(hex_to_bin(hexsha), typename, size, stream) From 5ca0fbae5c8f1ae112ddd0d4647d3399fe610501 Mon Sep 17 00:00:00 2001 From: jcole-crowdstrike Date: Fri, 1 Mar 2024 11:55:32 -0800 Subject: [PATCH 0726/1392] Updating regex pattern to handle unicode whitespaces. Replacing the \s whitespace characters with normal spaces (" ") to prevent breaking on unicode whitespace characters (e.g., NBSP). Without this, if a branch name contains a unicode whitespace character that falls under \s, then the branch name will be truncated. --- git/remote.py | 2 +- test/test_remote.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/git/remote.py b/git/remote.py index 6ce720ee3..6e841949f 100644 --- a/git/remote.py +++ b/git/remote.py @@ -325,7 +325,7 @@ class FetchInfo(IterableObj): ERROR, ) = [1 << x for x in range(8)] - _re_fetch_result = re.compile(r"^\s*(.) (\[[\w\s\.$@]+\]|[\w\.$@]+)\s+(.+) -> ([^\s]+)( \(.*\)?$)?") + _re_fetch_result = re.compile(r"^ *(.) (\[[\w \.$@]+\]|[\w\.$@]+) +(.+) -> ([^ ]+)( \(.*\)?$)?") _flag_map: Dict[flagKeyLiteral, int] = { "!": ERROR, diff --git a/test/test_remote.py b/test/test_remote.py index c0bd11f80..63eababac 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -1002,6 +1002,21 @@ def test_push_unsafe_options_allowed(self, rw_repo): assert tmp_file.exists() tmp_file.unlink() + @with_rw_and_rw_remote_repo("0.1.6") + def test_fetch_unsafe_branch_name(self, rw_repo, remote_repo): + # Create branch with a name containing a NBSP + bad_branch_name = f"branch_with_{chr(160)}_nbsp" + Head.create(remote_repo, bad_branch_name) + + # Fetch and get branches + remote = rw_repo.remote("origin") + branches = remote.fetch() + + # Test for truncated branch name in branches + assert f"origin/{bad_branch_name}" in [b.name for b in branches] + + # Cleanup branch + Head.delete(remote_repo, bad_branch_name) class TestTimeouts(TestBase): @with_rw_repo("HEAD", bare=False) From 827e986ec686b168e65f3bbad526d6a863191031 Mon Sep 17 00:00:00 2001 From: jcole-crowdstrike Date: Fri, 1 Mar 2024 13:51:33 -0800 Subject: [PATCH 0727/1392] Updating spacing per linting test. Adding a second blank line after the newly added test case. --- test/test_remote.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_remote.py b/test/test_remote.py index 63eababac..35af8172d 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -1018,6 +1018,7 @@ def test_fetch_unsafe_branch_name(self, rw_repo, remote_repo): # Cleanup branch Head.delete(remote_repo, bad_branch_name) + class TestTimeouts(TestBase): @with_rw_repo("HEAD", bare=False) def test_timeout_funcs(self, repo): From fc59e5eafde730c9380266335edb278a23630ee8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 1 Mar 2024 18:59:39 -0500 Subject: [PATCH 0728/1392] Improve stream_object_data and _parse_object_header docstrings This slightly adjusts working and formatting of these methods of the Git class, for clarity. --- git/cmd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 4fd6dc7cd..41b88bd54 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1491,7 +1491,9 @@ def _call_process( def _parse_object_header(self, header_line: str) -> Tuple[str, str, int]: """ :param header_line: - type_string size_as_int + A line of the form:: + + type_string size_as_int :return: (hex_sha, type_string, size_as_int) @@ -1581,7 +1583,7 @@ def get_object_data(self, ref: str) -> Tuple[str, str, int, bytes]: return (hexsha, typename, size, data) def stream_object_data(self, ref: str) -> Tuple[str, str, int, "Git.CatFileContentStream"]: - """Similar to :meth:`get_object_header`, but returns the data as a stream. + """Similar to :meth:`get_object_data`, but returns the data as a stream. :return: (hexsha, type_string, size_as_int, stream) From 7044ff6b9c5381c5506fd1f2a71546641ed259f7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 2 Mar 2024 01:48:30 -0500 Subject: [PATCH 0729/1392] Include top-level git.refresh in API Reference For #1854. --- doc/source/reference.rst | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/doc/source/reference.rst b/doc/source/reference.rst index 68a7f0ba4..6ee25c04d 100644 --- a/doc/source/reference.rst +++ b/doc/source/reference.rst @@ -3,13 +3,16 @@ API Reference ============= -Version -------- +Top-Level +--------- .. py:data:: git.__version__ Current GitPython version. +.. automodule:: git + :members: refresh + Objects.Base ------------ @@ -17,7 +20,7 @@ Objects.Base :members: :undoc-members: :special-members: - + Objects.Blob ------------ @@ -25,7 +28,7 @@ Objects.Blob :members: :undoc-members: :special-members: - + Objects.Commit -------------- @@ -33,7 +36,7 @@ Objects.Commit :members: :undoc-members: :special-members: - + Objects.Tag ----------- @@ -73,7 +76,7 @@ Objects.Submodule.root :members: :undoc-members: :special-members: - + Objects.Submodule.util ---------------------- @@ -81,7 +84,7 @@ Objects.Submodule.util :members: :undoc-members: :special-members: - + Objects.Util ------------- @@ -105,7 +108,7 @@ Index.Functions :members: :undoc-members: :special-members: - + Index.Types ----------- @@ -113,7 +116,7 @@ Index.Types :members: :undoc-members: :special-members: - + Index.Util ------------- @@ -121,7 +124,7 @@ Index.Util :members: :undoc-members: :special-members: - + GitCmd ------ @@ -137,7 +140,7 @@ Config :members: :undoc-members: :special-members: - + Diff ---- @@ -154,7 +157,7 @@ Exceptions :undoc-members: :special-members: - + Refs.symbolic ------------- @@ -162,7 +165,7 @@ Refs.symbolic :members: :undoc-members: :special-members: - + Refs.reference -------------- @@ -178,7 +181,7 @@ Refs.head :members: :undoc-members: :special-members: - + Refs.tag ------------ @@ -186,7 +189,7 @@ Refs.tag :members: :undoc-members: :special-members: - + Refs.remote ------------ @@ -194,7 +197,7 @@ Refs.remote :members: :undoc-members: :special-members: - + Refs.log ------------ @@ -202,7 +205,7 @@ Refs.log :members: :undoc-members: :special-members: - + Remote ------ @@ -218,7 +221,7 @@ Repo.Base :members: :undoc-members: :special-members: - + Repo.Functions -------------- From 24160d1c7217622ec9b4de49ee2bf710e385ba0e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 2 Mar 2024 03:11:15 -0500 Subject: [PATCH 0730/1392] Add git.compat, git.db, and git.types in API Reference For #1854 (along with 7044ff6). --- doc/source/reference.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/source/reference.rst b/doc/source/reference.rst index 6ee25c04d..13dd38d02 100644 --- a/doc/source/reference.rst +++ b/doc/source/reference.rst @@ -230,6 +230,30 @@ Repo.Functions :undoc-members: :special-members: +Compat +------ + +.. automodule:: git.compat + :members: + :undoc-members: + :special-members: + +DB +-- + +.. automodule:: git.db + :members: + :undoc-members: + :special-members: + +Types +----- + +.. automodule:: git.types + :members: + :undoc-members: + :special-members: + Util ---- From d1da48d512a9f68ced06e47998295c6b910eb640 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 2 Mar 2024 03:14:03 -0500 Subject: [PATCH 0731/1392] Fix unterminated double-backtick in a git.compat docstring Adding git.compat in reference.rst revealed this (and makes this fix necessary for the documentation to build). --- git/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/compat.py b/git/compat.py index 3167cecdc..7753fe8b2 100644 --- a/git/compat.py +++ b/git/compat.py @@ -61,7 +61,7 @@ :note: For macOS (Darwin), ``os.name == "posix"`` as in other Unix-like systems, while - ``sys.platform == "darwin"`. + ``sys.platform == "darwin"``. """ defenc = sys.getfilesystemencoding() From c4a6618eefcc5c2610bb0e247e02d188ac10663f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 2 Mar 2024 11:14:03 +0100 Subject: [PATCH 0732/1392] Remove duplicate information in docstring --- git/cmd.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 41b88bd54..c7cec48d7 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -393,9 +393,6 @@ def refresh(cls, path: Union[None, PathLike] = None) -> bool: Called by the :func:`git.refresh` function in the top level ``__init__``. - This gets called by the :func:`git.refresh` function in the top-level - ``__init__``. - :param path: Optional path to the git executable. If not absolute, it is resolved immediately, relative to the current directory. (See note below.) From 5253b8d6910cf2defea946955f86ea80ffcac8a2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 2 Mar 2024 03:46:00 -0500 Subject: [PATCH 0733/1392] Restore building of documentation downloads Although documentation resumed being built in 634151a, and the RTD theme and API Reference section were restored in 64ad585 (#1843), documentation for download did not resume being built, with 3.1.37 (not 3.1.42) being the latest version listed here: https://readthedocs.org/projects/gitpython/downloads/ Three kinds of downloadable documentation are supported -- PDF, ePub, and HTML -- and all three were previously being built but all have stopped. This attempts to resume building all three, using `all` as the value of the `formats` key in .readthedocs.yml. A string value of `all` currently should have the same effect as a sequence value of the strings `htmlzip`, `pdf`, and `epub`. (In the future, `all` may build more formats.) See: - https://docs.readthedocs.io/en/stable/downloadable-documentation.html - https://docs.readthedocs.io/en/stable/config-file/v2.html#formats --- .readthedocs.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 04275ce72..0b83e20ea 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -23,9 +23,7 @@ sphinx: fail_on_warning: true # Optionally build your docs in additional formats such as PDF and ePub. -# formats: -# - pdf -# - epub +formats: all # Optional but recommended, declare the Python requirements required # to build your documentation. From f83b056e6d89c45df8f3fb57fa3e0c6dbc16772b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 2 Mar 2024 15:29:12 -0500 Subject: [PATCH 0734/1392] Revise assert_never This revises the docstring of git.types.assert_never to more clearly express its semantics and usage, to use the type names used in the typing module, typing_extensions package, and mypy. This expands some parts when doing so seemed to benefit clarity. The docstring had previously said AssertionError was raised, but the code raised ValueError. For now I've adjusted the docstring to be accurate to the code's behavior, but maybe this should change. I also removed the part about the mypy error being on variable creation, since I am not clear on what that was referring to, and if it means the first name binding operation for a variable in its scope, then I am not sure why that would be where an error message would be expected when using assert_never. Finally, this slightly adjusts the message: - "Literal" is decapitalized, to decrease confusion if the default message is used when matching on something not a typing.Literal. - The exception message prints the unexpected value's repr rather than its str, since the repr, when different from the str, is usually the representation more useful for debugging. --- git/types.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/git/types.py b/git/types.py index efb393471..0bd723176 100644 --- a/git/types.py +++ b/git/types.py @@ -72,19 +72,27 @@ def assert_never(inp: NoReturn, raise_error: bool = True, exc: Union[Exception, None] = None) -> None: - """For use in exhaustive checking of literal or Enum in if/else chain. + """For use in exhaustive checking of a literal or enum in if/else chains. - Should only be reached if all members not handled OR attempt to pass non-members through chain. + A call to this function should only be reached if not all members are handled, or if + an attempt is made to pass non-members through the chain. - If all members handled, type is Empty. Otherwise, will cause mypy error. + :param inp: + If all members are handled, the argument for `inp` will have the + :class:`~typing.Never`/:class:`~typing.NoReturn` type. Otherwise, the type will + mismatch and cause a mypy error. - If non-members given, should cause mypy error at variable creation. + :param raise_error: + If ``True``, will also raise :class:`ValueError` with a general "unhandled + literal" message, or the exception object passed as `exc`. - If raise_error is True, will also raise AssertionError or the Exception passed to exc. + :param exc: + It not ``None``, this should be an already-constructed exception object, to be + raised if `raise_error` is ``True``. """ if raise_error: if exc is None: - raise ValueError(f"An unhandled Literal ({inp}) in an if/else chain was found") + raise ValueError(f"An unhandled literal ({inp!r}) in an if/else chain was found") else: raise exc From 01cc8e28ae1677286e32f9f00764dbefe83c3787 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 2 Mar 2024 21:40:30 -0500 Subject: [PATCH 0735/1392] Fix unnecessarily long reference in Tree docstrings There is no advantage to fully qualifying names in the same module as the entity being documented, but I had accidentally done that. --- git/objects/tree.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/objects/tree.py b/git/objects/tree.py index a506bba7d..a3bd7181c 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -162,7 +162,7 @@ def __delitem__(self, name: str) -> None: class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): R"""Tree objects represent an ordered list of :class:`~git.objects.blob.Blob`\s and - other :class:`~git.objects.tree.Tree`\s. + other :class:`Tree`\s. Tree as a list: @@ -230,8 +230,8 @@ def join(self, file: str) -> IndexObjUnion: """Find the named object in this tree's contents. :return: - :class:`~git.objects.blob.Blob`, :class:`~git.objects.tree.Tree`, - or :class:`~git.objects.submodule.base.Submodule` + :class:`~git.objects.blob.Blob`, :class:`Tree`, or + :class:`~git.objects.submodule.base.Submodule` :raise KeyError: If the given file or tree does not exist in this tree. From 6f3a20f1b651a75d38366f177708a96dd78d157e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 2 Mar 2024 21:45:42 -0500 Subject: [PATCH 0736/1392] Change how tree[subscript] is introduced This modifies the Tree docstring to replace the phrase "Tree as a list" with a phrase that: - Encompasses the use where subscripts are not integers or slices. - Is more precise in its terminology. The former advantage--not leaving out the case where one can pass a string--is the rationale. --- git/objects/tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/tree.py b/git/objects/tree.py index a3bd7181c..0960215aa 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -164,7 +164,7 @@ class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): R"""Tree objects represent an ordered list of :class:`~git.objects.blob.Blob`\s and other :class:`Tree`\s. - Tree as a list: + Subscripting is supported, as with a mapping or sequence: * Access a specific blob using the ``tree["filename"]`` notation. * You may likewise access by index, like ``blob = tree[0]``. From 85889cde01292c2c4b041eb031f645a5fdf62f27 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 2 Mar 2024 21:50:23 -0500 Subject: [PATCH 0737/1392] Refine how tree[subscript] is introduced This drops the newly introduced precision in Python terminology, likening usage to that of a list or dict (experienced Python users will already know about sequences and mappings, and inexperienced users will, with these terms, be able to understand what is said). Note that this does not undo the broader effect of the previous commit, which is to make clear that subscripting a tree supports both list-like and dict-like usage, not just list-like usage. --- git/objects/tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/tree.py b/git/objects/tree.py index 0960215aa..8e7ef6ae6 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -164,7 +164,7 @@ class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): R"""Tree objects represent an ordered list of :class:`~git.objects.blob.Blob`\s and other :class:`Tree`\s. - Subscripting is supported, as with a mapping or sequence: + Subscripting is supported, as with a list or dict: * Access a specific blob using the ``tree["filename"]`` notation. * You may likewise access by index, like ``blob = tree[0]``. From 9e470839fe0a082cfdb61ba18f31e5e18cd345ad Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 2 Mar 2024 23:39:29 -0500 Subject: [PATCH 0738/1392] Start adding docstrings to types in git.types --- git/types.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/git/types.py b/git/types.py index 0bd723176..0080e8911 100644 --- a/git/types.py +++ b/git/types.py @@ -39,6 +39,7 @@ # from typing_extensions import TypeGuard # noqa: F401 PathLike = Union[str, "os.PathLike[str]"] +"""A :class:`str` (Unicode) based file or directory path.""" if TYPE_CHECKING: from git.repo import Repo @@ -47,6 +48,8 @@ # from git.refs import SymbolicReference TBD = Any +"""Alias of :class:`~typing.Any`, when a type hint is meant to become more specific.""" + _T = TypeVar("_T") Tree_ish = Union["Commit", "Tree"] @@ -56,17 +59,27 @@ # Config_levels --------------------------------------------------------- Lit_config_levels = Literal["system", "global", "user", "repository"] +"""Type of literal strings naming git configuration levels. -# Progress parameter type alias ----------------------------------------- +Such a string identifies what level, or scope, a git configuration variables is in. +""" -CallableProgress = Optional[Callable[[int, Union[str, float], Union[str, float, None], str], None]] +ConfigLevels_Tup = Tuple[Literal["system"], Literal["user"], Literal["global"], Literal["repository"]] +"""Static type of a tuple of the four strings representing configuration levels.""" # def is_config_level(inp: str) -> TypeGuard[Lit_config_levels]: # # return inp in get_args(Lit_config_level) # only py >= 3.8 # return inp in ("system", "user", "global", "repository") +# Progress parameter type alias ----------------------------------------- -ConfigLevels_Tup = Tuple[Literal["system"], Literal["user"], Literal["global"], Literal["repository"]] +CallableProgress = Optional[Callable[[int, Union[str, float], Union[str, float, None], str], None]] +"""General type of a progress reporter for cloning. + +This is the type of a function or other callable that reports the progress of a clone, +when passed as a ``progress`` argument to :meth:`Repo.clone ` +or :meth:`Repo.clone_from `. +""" # ----------------------------------------------------------------------------------- From 3bd8177f07e724edfb780c8b19149fd19577cb88 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 2 Mar 2024 23:45:42 -0500 Subject: [PATCH 0739/1392] Document Tree_ish, Commit_ish, and related types The Commit_ish docstring may require substantive revision. The follow claims about the reasion for the design of Commit_ish should be checked, and if false removed or fixed, and if true then possibly clarified or otherwise refined: - "often usable where a commit-ish is expected" (both as for whether it is a reasonable generalization and for whether it is actually part of the reason Commit_ish includes Blob and Tree) - "It is done for practical reasons including backward compatibility." (for whether that is why it was done is or kept) --- git/types.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/git/types.py b/git/types.py index 0080e8911..497d5fa45 100644 --- a/git/types.py +++ b/git/types.py @@ -53,8 +53,58 @@ _T = TypeVar("_T") Tree_ish = Union["Commit", "Tree"] +"""Union of :class:`~git.objects.base.Object`-based types that are inherently tree-ish. + +See gitglossary(7) on "tree-ish": https://git-scm.com/docs/gitglossary#def_tree-ish + +:note: + This union comprises **only** the :class:`~git.objects.commit.Commit` and + :class:`~git.objects.tree.Tree` classes, **all** of whose instances are tree-ish. + This is done because of the way GitPython uses it as a static type annotation. + + :class:`~git.objects.tag.TagObject`, some but not all of whose instances are + tree-ish (those representing git tag objects that ultimately resolve to a tree or + commit), is not covered as part of this union type. +""" + Commit_ish = Union["Commit", "TagObject", "Blob", "Tree"] +"""Union of the :class:`~git.objects.base.Object`-based types that represent kinds of +git objects. This union is often usable where a commit-ish is expected, but is not +actually limited to types representing commit-ish git objects. + +See gitglossary(7) on: + +* "commit-ish": https://git-scm.com/docs/gitglossary#def_commit-ish +* "object type": https://git-scm.com/docs/gitglossary#def_object_type + +:note: + This union comprises **more** classes than those whose instances really represent + commit-ish git objects: + + * A :class:`~git.objects.commit.Commit` is of course always + commit-ish, and a :class:`~git.objects.tag.TagObject` is commit-ish if, when + peeled (recursively followed), a :class:`~git.objects.commit.Commit` is obtained. + * However, :class:`~git.objects.blob.Blob` and :class:`~git.objects.tree.Tree` are + also included, and they represent git objects that are never really commit-ish. + + This is an inversion of the situation with :class:`Tree_ish`, which is narrower than + all tree-ish objects. It is done for practical reasons including backward + compatibility. +""" + Lit_commit_ish = Literal["commit", "tag", "blob", "tree"] +"""Literal strings identifying concrete :class:`~git.objects.base.Object` subtypes. + +See :class:`Object.type `. + +:note: + See also :class:`Commit_ish`, a union of the the :class:`~git.objects.base.Object` + subtypes associated with these literal strings. + +:note: + As noted in :class:`Commit_ish`, this is not limited to types of git objects that + are actually commit-ish. +""" # Config_levels --------------------------------------------------------- From f3b9a695a066360c5dc0ce0209d8bd4a57126930 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 3 Mar 2024 00:06:09 -0500 Subject: [PATCH 0740/1392] Expand docs of classes representing Git objects This expands the docstrings of the Object base class, as well as the four leaf subclasses that represent git objects (Blob, Commit, TagObject, and Tree) to clarify those classes' relationship to git objects in general and specific types of git objects, to each other, and where applicable to types defined in git.types. This includes links to specific sections of the gitglossary(7) manpage (as hosted online), where directly applicable. --- git/objects/base.py | 53 ++++++++++++++++++++++++++++++++++++++----- git/objects/blob.py | 5 +++- git/objects/commit.py | 8 +++++-- git/objects/tag.py | 6 ++++- git/objects/tree.py | 3 +++ 5 files changed, 65 insertions(+), 10 deletions(-) diff --git a/git/objects/base.py b/git/objects/base.py index e78420e8a..c1df9e756 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -37,9 +37,33 @@ class Object(LazyMixin): - """An Object which may be :class:`~git.objects.blob.Blob`, - :class:`~git.objects.tree.Tree`, :class:`~git.objects.commit.Commit` or - `~git.objects.tag.TagObject`.""" + """Base class for classes representing kinds of git objects. + + The following leaf classes represent specific kinds of git objects: + + * :class:`Blob ` + * :class:`Tree ` + * :class:`Commit ` + * :class:`TagObject ` + + See gitglossary(7) on: + + * "object": https://git-scm.com/docs/gitglossary#def_object + * "object type": https://git-scm.com/docs/gitglossary#def_object_type + * "blob": https://git-scm.com/docs/gitglossary#def_blob_object + * "tree object": https://git-scm.com/docs/gitglossary#def_tree_object + * "commit object": https://git-scm.com/docs/gitglossary#def_commit_object + * "tag object": https://git-scm.com/docs/gitglossary#def_tag_object + + :note: + See the :class:`git.types.Commit_ish` union type. + + :note: + :class:`~git.objects.submodule.base.Submodule` is defined under the hierarchy + rooted at this :class:`Object` class, even though submodules are not really a + kind of git object. + + """ NULL_HEX_SHA = "0" * 40 NULL_BIN_SHA = b"\0" * 20 @@ -54,6 +78,20 @@ class Object(LazyMixin): __slots__ = ("repo", "binsha", "size") type: Union[Lit_commit_ish, None] = None + """String identifying (a concrete :class:`Object` subtype for) a kind of git object. + + The subtypes that this may name correspond to the kinds of git objects that exist, + i.e., the objects that may be present in a git repository. + + :note: + Most subclasses represent specific types of git objects and override this class + attribute accordingly. This attribute is ``None`` in the :class:`Object` base + class, as well as the :class:`IndexObject` intermediate subclass, but never + ``None`` in concrete leaf subclasses representing specific kinds of git objects. + + :note: + See also :class:`~git.types.Commit_ish`. + """ def __init__(self, repo: "Repo", binsha: bytes): """Initialize an object by identifying it by its binary sha. @@ -174,9 +212,12 @@ def stream_data(self, ostream: "OStream") -> "Object": class IndexObject(Object): - """Base for all objects that can be part of the index file, namely - :class:`~git.objects.tree.Tree`, :class:`~git.objects.blob.Blob` and - :class:`~git.objects.submodule.base.Submodule` objects.""" + """Base for all objects that can be part of the index file. + + The classes representing kinds of git objects that can be part of the index file are + :class:`~git.objects.tree.Tree and :class:`~git.objects.blob.Blob`. In addition, + :class:`~git.objects.submodule.base.Submodule` is also a subclass. + """ __slots__ = ("path", "mode") diff --git a/git/objects/blob.py b/git/objects/blob.py index 253ceccb5..b37570c3a 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -12,7 +12,10 @@ class Blob(base.IndexObject): - """A Blob encapsulates a git blob object.""" + """A Blob encapsulates a git blob object. + + See gitglossary(7) on "blob": https://git-scm.com/docs/gitglossary#def_blob_object + """ DEFAULT_MIME_TYPE = "text/plain" type: Literal["blob"] = "blob" diff --git a/git/objects/commit.py b/git/objects/commit.py index 06ab0898b..3246d9a36 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -60,8 +60,12 @@ class Commit(base.Object, TraversableIterableObj, Diffable, Serializable): """Wraps a git commit object. - This class will act lazily on some of its attributes and will query the value on - demand only if it involves calling the git binary. + See gitglossary(7) on "commit object": + https://git-scm.com/docs/gitglossary#def_commit_object + + :note: + This class will act lazily on some of its attributes and will query the value on + demand only if it involves calling the git binary. """ # ENVIRONMENT VARIABLES diff --git a/git/objects/tag.py b/git/objects/tag.py index f455c55fc..49fb97e37 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -30,7 +30,11 @@ class TagObject(base.Object): """Annotated (i.e. non-lightweight) tag carrying additional information about an - object we are pointing to.""" + object we are pointing to. + + See gitglossary(7) on "tag object": + https://git-scm.com/docs/gitglossary#def_tag_object + """ type: Literal["tag"] = "tag" diff --git a/git/objects/tree.py b/git/objects/tree.py index 8e7ef6ae6..995c9a663 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -164,6 +164,9 @@ class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): R"""Tree objects represent an ordered list of :class:`~git.objects.blob.Blob`\s and other :class:`Tree`\s. + See gitglossary(7) on "tree object": + https://git-scm.com/docs/gitglossary#def_tree_object + Subscripting is supported, as with a list or dict: * Access a specific blob using the ``tree["filename"]`` notation. From 2af7640345697888cce7c6eae7ab247aed2e497f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 3 Mar 2024 00:17:07 -0500 Subject: [PATCH 0741/1392] Do a bit of tidying related to unused names In git.types. - Remove commented-out import of SymbolicReference, which is not used anywhere, nor mentioned in commented-out code. - Add a docstring to _T to note that it is used within GitPython. (Otherwise it looks left over, as no code in git.types uses it.) --- git/types.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/git/types.py b/git/types.py index 497d5fa45..714a084ef 100644 --- a/git/types.py +++ b/git/types.py @@ -45,12 +45,11 @@ from git.repo import Repo from git.objects import Commit, Tree, TagObject, Blob - # from git.refs import SymbolicReference - TBD = Any """Alias of :class:`~typing.Any`, when a type hint is meant to become more specific.""" _T = TypeVar("_T") +"""Type variable used internally in GitPython.""" Tree_ish = Union["Commit", "Tree"] """Union of :class:`~git.objects.base.Object`-based types that are inherently tree-ish. From 2aa053e0d1eead06dce588265fdc8c0a1f29f7cd Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 3 Mar 2024 01:11:22 -0500 Subject: [PATCH 0742/1392] Add docstrings to TypedDicts in git.types --- git/types.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/git/types.py b/git/types.py index 714a084ef..b888cea51 100644 --- a/git/types.py +++ b/git/types.py @@ -160,12 +160,24 @@ def assert_never(inp: NoReturn, raise_error: bool = True, exc: Union[Exception, class Files_TD(TypedDict): + """Dictionary with stat counts for the diff of a particular file. + + For the :class:`~git.util.Stats.files` attribute of :class:`~git.util.Stats` + objects. + """ + insertions: int deletions: int lines: int class Total_TD(TypedDict): + """Dictionary with total stats from any number of files. + + For the :class:`~git.util.Stats.total` attribute of :class:`~git.util.Stats` + objects. + """ + insertions: int deletions: int lines: int @@ -173,6 +185,8 @@ class Total_TD(TypedDict): class HSH_TD(TypedDict): + """Dictionary carrying the same information as a :class:`~git.util.Stats` object.""" + total: Total_TD files: Dict[PathLike, Files_TD] From 15d50dee206c6e834ad9bea53ab182de381d7d41 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 4 Mar 2024 10:34:44 -0500 Subject: [PATCH 0743/1392] Revise a couple new docstrings for clarity --- git/types.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/git/types.py b/git/types.py index b888cea51..cb29be478 100644 --- a/git/types.py +++ b/git/types.py @@ -80,9 +80,9 @@ This union comprises **more** classes than those whose instances really represent commit-ish git objects: - * A :class:`~git.objects.commit.Commit` is of course always - commit-ish, and a :class:`~git.objects.tag.TagObject` is commit-ish if, when - peeled (recursively followed), a :class:`~git.objects.commit.Commit` is obtained. + * A :class:`~git.objects.commit.Commit` is of course always commit-ish, and a + :class:`~git.objects.tag.TagObject` is commit-ish if, when peeled (recursively + followed), a :class:`~git.objects.commit.Commit` is obtained. * However, :class:`~git.objects.blob.Blob` and :class:`~git.objects.tree.Tree` are also included, and they represent git objects that are never really commit-ish. @@ -92,9 +92,10 @@ """ Lit_commit_ish = Literal["commit", "tag", "blob", "tree"] -"""Literal strings identifying concrete :class:`~git.objects.base.Object` subtypes. +"""Literal strings identifying concrete :class:`~git.objects.base.Object` subtypes +representing kinds of git objects. -See :class:`Object.type `. +See the :class:`Object.type ` attribute. :note: See also :class:`Commit_ish`, a union of the the :class:`~git.objects.base.Object` @@ -110,7 +111,7 @@ Lit_config_levels = Literal["system", "global", "user", "repository"] """Type of literal strings naming git configuration levels. -Such a string identifies what level, or scope, a git configuration variables is in. +Such a string identifies what level, or scope, a git configuration variable is in. """ ConfigLevels_Tup = Tuple[Literal["system"], Literal["user"], Literal["global"], Literal["repository"]] From 716670326d5fba5356628408237dfb7786d5d228 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 4 Mar 2024 10:40:26 -0500 Subject: [PATCH 0744/1392] Fix possible inaccuracy in Lit_config_levels docstring These are not necessarily what one means by the "scope" of a configuration variable, in part because of the nature of the subtle distinction between "user" and "global", and in part because of other issues such as how setting a variable for a single command with "-c" has a scope which is not listed. This also brings the docstring somewhat more in line with how these values are documented elsewhere in GitPython. --- git/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/types.py b/git/types.py index cb29be478..9cd38a4f4 100644 --- a/git/types.py +++ b/git/types.py @@ -111,7 +111,7 @@ Lit_config_levels = Literal["system", "global", "user", "repository"] """Type of literal strings naming git configuration levels. -Such a string identifies what level, or scope, a git configuration variable is in. +These strings relate to which file a git configuration variable is in. """ ConfigLevels_Tup = Tuple[Literal["system"], Literal["user"], Literal["global"], Literal["repository"]] From 1530fd2b8137bb7c93206cdb373374db46c884c1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 4 Mar 2024 10:52:05 -0500 Subject: [PATCH 0745/1392] Use phrases like "git object type" where applicable In some recently introduced or expanded docstrings, I had overused phrases like "kind of git object" with the hope of avoiding confusion with the meanings of "type" relevant to Python (i.e., "class" or "static type"). But this made the relationship to git's own notion of "object type" less clear than it could be, especially in docstrings that also included links to the gitglossary(7) entry for "object type" (because those links' relevance was less clear). This dials back the use of "kind" to where I am more confident that it is clarifying or at least not confusing. --- git/objects/base.py | 13 +++++++------ git/types.py | 8 ++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/git/objects/base.py b/git/objects/base.py index c1df9e756..0496b18cd 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -37,7 +37,7 @@ class Object(LazyMixin): - """Base class for classes representing kinds of git objects. + """Base class for classes representing git object types. The following leaf classes represent specific kinds of git objects: @@ -61,7 +61,7 @@ class Object(LazyMixin): :note: :class:`~git.objects.submodule.base.Submodule` is defined under the hierarchy rooted at this :class:`Object` class, even though submodules are not really a - kind of git object. + type of git object. """ @@ -78,7 +78,7 @@ class Object(LazyMixin): __slots__ = ("repo", "binsha", "size") type: Union[Lit_commit_ish, None] = None - """String identifying (a concrete :class:`Object` subtype for) a kind of git object. + """String identifying (a concrete :class:`Object` subtype for) a git object type. The subtypes that this may name correspond to the kinds of git objects that exist, i.e., the objects that may be present in a git repository. @@ -87,7 +87,7 @@ class Object(LazyMixin): Most subclasses represent specific types of git objects and override this class attribute accordingly. This attribute is ``None`` in the :class:`Object` base class, as well as the :class:`IndexObject` intermediate subclass, but never - ``None`` in concrete leaf subclasses representing specific kinds of git objects. + ``None`` in concrete leaf subclasses representing specific git object types. :note: See also :class:`~git.types.Commit_ish`. @@ -214,9 +214,10 @@ def stream_data(self, ostream: "OStream") -> "Object": class IndexObject(Object): """Base for all objects that can be part of the index file. - The classes representing kinds of git objects that can be part of the index file are + The classes representing git object types that can be part of the index file are :class:`~git.objects.tree.Tree and :class:`~git.objects.blob.Blob`. In addition, - :class:`~git.objects.submodule.base.Submodule` is also a subclass. + :class:`~git.objects.submodule.base.Submodule`, which is not really a git object + type but can be part of an index file, is also a subclass. """ __slots__ = ("path", "mode") diff --git a/git/types.py b/git/types.py index 9cd38a4f4..e8e8dd3e4 100644 --- a/git/types.py +++ b/git/types.py @@ -67,9 +67,9 @@ """ Commit_ish = Union["Commit", "TagObject", "Blob", "Tree"] -"""Union of the :class:`~git.objects.base.Object`-based types that represent kinds of -git objects. This union is often usable where a commit-ish is expected, but is not -actually limited to types representing commit-ish git objects. +"""Union of the :class:`~git.objects.base.Object`-based types that represent git object +types. This union is often usable where a commit-ish is expected, but is not actually +limited to types representing commit-ish git objects. See gitglossary(7) on: @@ -93,7 +93,7 @@ Lit_commit_ish = Literal["commit", "tag", "blob", "tree"] """Literal strings identifying concrete :class:`~git.objects.base.Object` subtypes -representing kinds of git objects. +representing git object types. See the :class:`Object.type ` attribute. From 2e02b09f1d9aeda2db1fe2340c131bb42ab87cb9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 4 Mar 2024 11:47:53 -0500 Subject: [PATCH 0746/1392] Add docstrings to protocols in git.types --- git/types.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/git/types.py b/git/types.py index e8e8dd3e4..9159195cd 100644 --- a/git/types.py +++ b/git/types.py @@ -194,9 +194,13 @@ class HSH_TD(TypedDict): @runtime_checkable class Has_Repo(Protocol): + """Protocol for having a :attr:`repo` attribute, the repository to operate on.""" + repo: "Repo" @runtime_checkable class Has_id_attribute(Protocol): + """Protocol for having :attr:`_id_attribute_` used in iteration and traversal.""" + _id_attribute_: str From 012d710c18ee59bafa9565f9d9ca5609a88d21f8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 4 Mar 2024 11:50:41 -0500 Subject: [PATCH 0747/1392] Move our PathLike below even TYPE_CHECKING imports The benefits are that putting imports before newly introduced names (other than names like __all__) is recommended by PEP-8, and that git.types.PathLike is not equivalent to os.PathLike and introducing it after all imports may help avoid obscuring this. --- git/types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/types.py b/git/types.py index 9159195cd..efcd2a7aa 100644 --- a/git/types.py +++ b/git/types.py @@ -38,13 +38,13 @@ # else: # from typing_extensions import TypeGuard # noqa: F401 -PathLike = Union[str, "os.PathLike[str]"] -"""A :class:`str` (Unicode) based file or directory path.""" - if TYPE_CHECKING: from git.repo import Repo from git.objects import Commit, Tree, TagObject, Blob +PathLike = Union[str, "os.PathLike[str]"] +"""A :class:`str` (Unicode) based file or directory path.""" + TBD = Any """Alias of :class:`~typing.Any`, when a type hint is meant to become more specific.""" From a06f1fc9834f8fc1feffd0249ef38c1cca302945 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 4 Mar 2024 11:53:16 -0500 Subject: [PATCH 0748/1392] Remove commented-out is_config_level function And the commented-out imports that had been solely to support it. --- git/types.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/git/types.py b/git/types.py index efcd2a7aa..7e1cceaa8 100644 --- a/git/types.py +++ b/git/types.py @@ -33,11 +33,6 @@ runtime_checkable, ) -# if sys.version_info >= (3, 10): -# from typing import TypeGuard # noqa: F401 -# else: -# from typing_extensions import TypeGuard # noqa: F401 - if TYPE_CHECKING: from git.repo import Repo from git.objects import Commit, Tree, TagObject, Blob @@ -117,10 +112,6 @@ ConfigLevels_Tup = Tuple[Literal["system"], Literal["user"], Literal["global"], Literal["repository"]] """Static type of a tuple of the four strings representing configuration levels.""" -# def is_config_level(inp: str) -> TypeGuard[Lit_config_levels]: -# # return inp in get_args(Lit_config_level) # only py >= 3.8 -# return inp in ("system", "user", "global", "repository") - # Progress parameter type alias ----------------------------------------- CallableProgress = Optional[Callable[[int, Union[str, float], Union[str, float, None], str], None]] From c93e431e73b896515999b6dbb4cf61ac6931fd37 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 4 Mar 2024 12:00:59 -0500 Subject: [PATCH 0749/1392] Expand git.compat docstring To make clear that code outside GitPython would not typically benefit from using anything in that module. See #1854 for context. --- git/compat.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/git/compat.py b/git/compat.py index 7753fe8b2..43b50b287 100644 --- a/git/compat.py +++ b/git/compat.py @@ -3,7 +3,12 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Utilities to help provide compatibility with Python 3.""" +"""Utilities to help provide compatibility with Python 3. + +This module exists for historical reasons. Code outside GitPython may make use of public +members of this module, but is unlikely to benefit from doing so. GitPython continues to +use some of these utilities, in some cases for compatibility across different platforms. +""" import locale import os From 29443ce94b3a687a5ab85edadafc231f8e5455ee Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 4 Mar 2024 14:57:01 -0500 Subject: [PATCH 0750/1392] Add a cationary note about Object vs. object See also f78587f (#1725). --- git/objects/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/git/objects/base.py b/git/objects/base.py index 0496b18cd..cb660b347 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -63,6 +63,9 @@ class Object(LazyMixin): rooted at this :class:`Object` class, even though submodules are not really a type of git object. + :note: + This :class:`Object` class should not be confused with :class:`object` (the root + of the class hierarchy in Python). """ NULL_HEX_SHA = "0" * 40 From b6e3ad2f9cf00e066ac40ec472143ec99b2f90db Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 4 Mar 2024 15:40:46 -0500 Subject: [PATCH 0751/1392] Don't bind unused _assertion_msg_format The _assertion_msg_format module attribute (global variable) of git.objects.base was formerly used in an assertion check that has since been commented out but not completely removed. It may be that both it and the commented-out code that uses it should simply be removed (they will be in the git history, after all), but this change just brings them in line by also commenting out the variable. --- git/objects/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/base.py b/git/objects/base.py index cb660b347..c1f2b0e63 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -31,7 +31,7 @@ # -------------------------------------------------------------------------- -_assertion_msg_format = "Created object %r whose python type %r disagrees with the actual git object type %r" +# _assertion_msg_format = "Created object %r whose python type %r disagrees with the actual git object type %r" __all__ = ("Object", "IndexObject") From 9f226fc47683038acd3304dd7d59be18984d3737 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 4 Mar 2024 19:38:10 -0500 Subject: [PATCH 0752/1392] Remove unneeded annotation on __slots__ variable The correct type is inferred, and no other __slots__ variable in the codebase has an annotation. --- git/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index c7cec48d7..8550830aa 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -749,7 +749,7 @@ class CatFileContentStream: rest to ensure the underlying stream continues to work. """ - __slots__: Tuple[str, ...] = ("_stream", "_nbr", "_size") + __slots__ = ("_stream", "_nbr", "_size") def __init__(self, size: int, stream: IO[bytes]) -> None: self._stream = stream From e984bfebd9f1934b9810ac6cccb1bb803c64dd21 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 4 Mar 2024 20:02:47 -0500 Subject: [PATCH 0753/1392] Fix some underindented portions of docstrings A few docstrings had parts that were meant to be indented the usual four spaces beyond the surrounding indentation, but were indented only three spaces beyond it instead. --- git/cmd.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 8550830aa..3d5992dbd 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -850,10 +850,10 @@ def __init__(self, working_dir: Union[None, PathLike] = None): """Initialize this instance with: :param working_dir: - Git directory we should work in. If ``None``, we always work in the current - directory as returned by :func:`os.getcwd`. - This is meant to be the working tree directory if available, or the - ``.git`` directory in case of bare repositories. + Git directory we should work in. If ``None``, we always work in the current + directory as returned by :func:`os.getcwd`. + This is meant to be the working tree directory if available, or the + ``.git`` directory in case of bare repositories. """ super().__init__() self._working_dir = expand_path(working_dir) @@ -1103,8 +1103,8 @@ def execute( :raise git.exc.GitCommandError: :note: - If you add additional keyword arguments to the signature of this method, - you must update the ``execute_kwargs`` variable housed in this module. + If you add additional keyword arguments to the signature of this method, you + must update the ``execute_kwargs`` variable housed in this module. """ # Remove password for the command if present. redacted_command = remove_password_if_present(command) @@ -1438,7 +1438,7 @@ def _call_process( turns into:: - git rev-list max-count 10 --header master + git rev-list max-count 10 --header master :return: Same as :meth:`execute`. If no args are given, used :meth:`execute`'s From 5d7e55b8dd2abaacd5397860e6efd7bd4215fe12 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 4 Mar 2024 20:11:40 -0500 Subject: [PATCH 0754/1392] Add return-type annotation on __init__ methods Although sometimes found unintuitive, the reutrn type of a class's __init__ method is best annotated as None, as in other functions that return implicitly or by operand-less return statements. Note that this is only a minor improvement, effectively just a style fix, because mypy treats __init__ specially and, *when at least one parameter is annotated*, its return type is implicitly None rather than implicitly Any like other functions. All the __init__ methods modified here did already have one or more annotated parameters. However, having __init__ methods without the return type makes it easier to introduce a bug where an __init__ method with no parameters besides self -- which itself should almost always be unannotated and is properly inferred -- is written and the return annotation needed to get mypy to regard it as statically typed, so it checks it at all, is omitted. (It is also inelegant when one considers the meaning of __init__ and the distinction between it and __new__.) This commit does not add any return type annotations to functions in the test suite, since the test suite doesn't currently use static typing. Further reading: - https://peps.python.org/pep-0484/#the-meaning-of-annotations - https://github.com/python/mypy/issues/604 - https://github.com/gitpython-developers/GitPython/pull/1755#issuecomment-1837419098 --- git/cmd.py | 2 +- git/objects/base.py | 2 +- git/objects/submodule/root.py | 2 +- git/refs/head.py | 2 +- git/refs/log.py | 2 +- git/refs/symbolic.py | 2 +- git/util.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 3d5992dbd..731d0fab8 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -846,7 +846,7 @@ def __del__(self) -> None: self._stream.read(bytes_left + 1) # END handle incomplete read - def __init__(self, working_dir: Union[None, PathLike] = None): + def __init__(self, working_dir: Union[None, PathLike] = None) -> None: """Initialize this instance with: :param working_dir: diff --git a/git/objects/base.py b/git/objects/base.py index e78420e8a..2b8dd0ff6 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -55,7 +55,7 @@ class Object(LazyMixin): type: Union[Lit_commit_ish, None] = None - def __init__(self, repo: "Repo", binsha: bytes): + def __init__(self, repo: "Repo", binsha: bytes) -> None: """Initialize an object by identifying it by its binary sha. All keyword arguments will be set on demand if ``None``. diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index ac0f7ad94..3268d73a4 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -56,7 +56,7 @@ class RootModule(Submodule): k_root_name = "__ROOT__" - def __init__(self, repo: "Repo"): + def __init__(self, repo: "Repo") -> None: # repo, binsha, mode=None, path=None, name = None, parent_commit=None, url=None, ref=None) super().__init__( repo, diff --git a/git/refs/head.py b/git/refs/head.py index d546cb4ef..f6020f461 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -44,7 +44,7 @@ class HEAD(SymbolicReference): __slots__ = () - def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME): + def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME) -> None: if path != self._HEAD_NAME: raise ValueError("HEAD instance must point to %r, got %r" % (self._HEAD_NAME, path)) super().__init__(repo, path) diff --git a/git/refs/log.py b/git/refs/log.py index 7aefeb4e6..f98f56f11 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -164,7 +164,7 @@ def __new__(cls, filepath: Union[PathLike, None] = None) -> "RefLog": inst = super().__new__(cls) return inst - def __init__(self, filepath: Union[PathLike, None] = None): + def __init__(self, filepath: Union[PathLike, None] = None) -> None: """Initialize this instance with an optional filepath, from which we will initialize our data. The path is also used to write changes back using the :meth:`write` method.""" diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 142952e06..16aada0a7 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -74,7 +74,7 @@ class SymbolicReference: _remote_common_path_default = "refs/remotes" _id_attribute_ = "name" - def __init__(self, repo: "Repo", path: PathLike, check_path: bool = False): + def __init__(self, repo: "Repo", path: PathLike, check_path: bool = False) -> None: self.repo = repo self.path = path diff --git a/git/util.py b/git/util.py index 52f9dbdad..d272dc53c 100644 --- a/git/util.py +++ b/git/util.py @@ -917,7 +917,7 @@ class Stats: __slots__ = ("total", "files") - def __init__(self, total: Total_TD, files: Dict[PathLike, Files_TD]): + def __init__(self, total: Total_TD, files: Dict[PathLike, Files_TD]) -> None: self.total = total self.files = files From 4810491cb9d30d05e3ed73e55fe803fcf7ae0b53 Mon Sep 17 00:00:00 2001 From: jcole-crowdstrike Date: Mon, 4 Mar 2024 20:33:24 -0800 Subject: [PATCH 0755/1392] Fixing Windows encoding issue. Stdout is being encoded as Windows-1251 instead of UTF-8 due to passing universal_newlines as True to subprocess. --- git/remote.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/remote.py b/git/remote.py index 6e841949f..6cb33bede 100644 --- a/git/remote.py +++ b/git/remote.py @@ -891,7 +891,7 @@ def _get_fetch_info_from_stderr( None, progress_handler, finalizer=None, - decode_streams=False, + decode_streams=True, kill_after_timeout=kill_after_timeout, ) @@ -1068,7 +1068,7 @@ def fetch( Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_fetch_options) proc = self.repo.git.fetch( - "--", self, *args, as_process=True, with_stdout=False, universal_newlines=True, v=verbose, **kwargs + "--", self, *args, as_process=True, with_stdout=False, v=verbose, **kwargs ) res = self._get_fetch_info_from_stderr(proc, progress, kill_after_timeout=kill_after_timeout) if hasattr(self.repo.odb, "update_cache"): From b5d91987a2ad160ab5ca7e5e5ca95bf332536549 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 6 Mar 2024 14:09:38 -0500 Subject: [PATCH 0756/1392] Remove commented-out code --- git/objects/base.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/git/objects/base.py b/git/objects/base.py index c1f2b0e63..215557db5 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -30,9 +30,6 @@ # -------------------------------------------------------------------------- - -# _assertion_msg_format = "Created object %r whose python type %r disagrees with the actual git object type %r" - __all__ = ("Object", "IndexObject") @@ -154,8 +151,7 @@ def _set_cache_(self, attr: str) -> None: """Retrieve object information.""" if attr == "size": oinfo = self.repo.odb.info(self.binsha) - self.size = oinfo.size # type: int - # assert oinfo.type == self.type, _assertion_msg_format % (self.binsha, oinfo.type, self.type) + self.size = oinfo.size # type: int else: super()._set_cache_(attr) From 2212ac98912b66eb848cd60e7f5c43067dc0c1d7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Mar 2024 00:44:36 -0500 Subject: [PATCH 0757/1392] Fix Sphinx reference that rendered overly long --- git/objects/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/base.py b/git/objects/base.py index 215557db5..df9b2d9f1 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -53,7 +53,7 @@ class Object(LazyMixin): * "tag object": https://git-scm.com/docs/gitglossary#def_tag_object :note: - See the :class:`git.types.Commit_ish` union type. + See the :class:`~git.types.Commit_ish` union type. :note: :class:`~git.objects.submodule.base.Submodule` is defined under the hierarchy From 3c5ca52b2462964e1d478ec796dd7c5dd8bb2f68 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Mar 2024 00:44:59 -0500 Subject: [PATCH 0758/1392] Simplify _safer_popen_windows "if shell" logic This fixes a static typing error reported by mypy. Using a separate variable, as I had done originally, does not seem to be clearer than rebinding the parameter, and in this case the code is simpler and can be checked by mypy without needing another explicit type annotation to be added. This fixes one mypy error. This also adds a comment to make clearer why the mapping passed in is copied when modifications are needed, rather than mutated. --- git/cmd.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index c7cec48d7..6354da666 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -257,10 +257,9 @@ def _safer_popen_windows( # When using a shell, the shell is the direct subprocess, so the variable must be # set in its environment, to affect its search behavior. (The "1" can be any value.) if shell: - safer_env = {} if env is None else dict(env) - safer_env["NoDefaultCurrentDirectoryInExePath"] = "1" - else: - safer_env = env + # The original may be immutable or reused by the caller. Make changes in a copy. + env = {} if env is None else dict(env) + env["NoDefaultCurrentDirectoryInExePath"] = "1" # When not using a shell, the current process does the search in a CreateProcessW # API call, so the variable must be set in our environment. With a shell, this is @@ -273,7 +272,7 @@ def _safer_popen_windows( return Popen( command, shell=shell, - env=safer_env, + env=env, creationflags=creationflags, **kwargs, ) From 43b7f8a3eb299c550674c6f95c9919019dc5a194 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Mar 2024 10:31:31 -0500 Subject: [PATCH 0759/1392] Annotate safer_popen broad enough for all platforms This fixes another static typing error reported by mypy. (The annotation could be made more specific in the future by making a custom protocol for it, which may or may not be worthwhile, given that `**kwargs: Any` would still have to be present after whatever typed keyword arguments the protocol's `__call__` method listed, since some callers intentionally forward arbitrary extra keyword arguments through safer_popen to Popen.) --- git/cmd.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git/cmd.py b/git/cmd.py index 6354da666..c37d28988 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -278,6 +278,8 @@ def _safer_popen_windows( ) +safer_popen: Callable[..., Popen] + if os.name == "nt": safer_popen = _safer_popen_windows else: From dc95a768b3381cf5a68fc9ad4e8dc4bba37b0ca0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Mar 2024 11:34:11 -0500 Subject: [PATCH 0760/1392] Fix mypy error with creationflags in subprocess module On Windows, Python's subprocess module contains constants useful to pass to the `creationflags` parameter of subprocess.Popen. These are absent on other platforms, where they are not meaningful. The code was already safe at runtime from any AttributeError related to these constants, because they were only used in git.cmd._safer_popen_windows, which was defined on Windows. But in #1792 I did not write the code in a way where mypy can verify its correctness. So if a regression of the kind mypy can in principle catch were to occur, it would be harder to notice it. This refactors the code, keeping the same behavior but expressing it in a way mypy can understand. This consists of two changes: 1. Only define _safer_popen_windows when the platform is Windows, placing it in the first branch of the `if` statement. This is needed because mypy will not take the only current call to that nonpublic function being on Windows as sufficient evidence that the platform is always Windows when it is run. 2. Determine the platform, for this purpose, using sys.platform instead of os.name. These are far from equivalent in general (see the deprecation rationale for is_ in #1732, revised in a0fa2bd in #1787). However, in Python 3 (GitPython no longer supports Python 2), in the specific case of Windows, we have a choice of which to use, as both `sys.platform == "win32"` and `os.name == "nt"`. os.name is "nt" on native Windows, and "posix" on Cygwin. sys.platform is "win32" on native Windows (including 64-bit systems with 64-bit Python builds), and "cygwin" on Cygwin. See: https://docs.python.org/3/library/sys.html#sys.platform This is needed because the type stubs for the subprocess module use this sys.platform check (rather than an os.name check) to determine if the platform is Windows for the purpose of deciding which constants to say the subprocess module defines. I have verified that neither of these changes is enough by itself. --- git/cmd.py | 113 +++++++++++++++++++++++++++-------------------------- 1 file changed, 57 insertions(+), 56 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index c37d28988..04037c06f 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -14,6 +14,7 @@ import signal from subprocess import Popen, PIPE, DEVNULL import subprocess +import sys import threading from textwrap import dedent @@ -220,67 +221,67 @@ def pump_stream( finalizer(process) -def _safer_popen_windows( - command: Union[str, Sequence[Any]], - *, - shell: bool = False, - env: Optional[Mapping[str, str]] = None, - **kwargs: Any, -) -> Popen: - """Call :class:`subprocess.Popen` on Windows but don't include a CWD in the search. - - This avoids an untrusted search path condition where a file like ``git.exe`` in a - malicious repository would be run when GitPython operates on the repository. The - process using GitPython may have an untrusted repository's working tree as its - current working directory. Some operations may temporarily change to that directory - before running a subprocess. In addition, while by default GitPython does not run - external commands with a shell, it can be made to do so, in which case the CWD of - the subprocess, which GitPython usually sets to a repository working tree, can - itself be searched automatically by the shell. This wrapper covers all those cases. +safer_popen: Callable[..., Popen] - :note: - This currently works by setting the :envvar:`NoDefaultCurrentDirectoryInExePath` - environment variable during subprocess creation. It also takes care of passing - Windows-specific process creation flags, but that is unrelated to path search. +if sys.platform == "win32": - :note: - The current implementation contains a race condition on :attr:`os.environ`. - GitPython isn't thread-safe, but a program using it on one thread should ideally - be able to mutate :attr:`os.environ` on another, without unpredictable results. - See comments in https://github.com/gitpython-developers/GitPython/pull/1650. - """ - # CREATE_NEW_PROCESS_GROUP is needed for some ways of killing it afterwards. See: - # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal - # https://docs.python.org/3/library/subprocess.html#subprocess.CREATE_NEW_PROCESS_GROUP - creationflags = subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP - - # When using a shell, the shell is the direct subprocess, so the variable must be - # set in its environment, to affect its search behavior. (The "1" can be any value.) - if shell: - # The original may be immutable or reused by the caller. Make changes in a copy. - env = {} if env is None else dict(env) - env["NoDefaultCurrentDirectoryInExePath"] = "1" - - # When not using a shell, the current process does the search in a CreateProcessW - # API call, so the variable must be set in our environment. With a shell, this is - # unnecessary, in versions where https://github.com/python/cpython/issues/101283 is - # patched. If that is unpatched, then in the rare case the ComSpec environment - # variable is unset, the search for the shell itself is unsafe. Setting - # NoDefaultCurrentDirectoryInExePath in all cases, as is done here, is simpler and - # protects against that. (As above, the "1" can be any value.) - with patch_env("NoDefaultCurrentDirectoryInExePath", "1"): - return Popen( - command, - shell=shell, - env=env, - creationflags=creationflags, - **kwargs, - ) + def _safer_popen_windows( + command: Union[str, Sequence[Any]], + *, + shell: bool = False, + env: Optional[Mapping[str, str]] = None, + **kwargs: Any, + ) -> Popen: + """Call :class:`subprocess.Popen` on Windows but don't include a CWD in the search. + + This avoids an untrusted search path condition where a file like ``git.exe`` in a + malicious repository would be run when GitPython operates on the repository. The + process using GitPython may have an untrusted repository's working tree as its + current working directory. Some operations may temporarily change to that directory + before running a subprocess. In addition, while by default GitPython does not run + external commands with a shell, it can be made to do so, in which case the CWD of + the subprocess, which GitPython usually sets to a repository working tree, can + itself be searched automatically by the shell. This wrapper covers all those cases. + :note: + This currently works by setting the :envvar:`NoDefaultCurrentDirectoryInExePath` + environment variable during subprocess creation. It also takes care of passing + Windows-specific process creation flags, but that is unrelated to path search. -safer_popen: Callable[..., Popen] + :note: + The current implementation contains a race condition on :attr:`os.environ`. + GitPython isn't thread-safe, but a program using it on one thread should ideally + be able to mutate :attr:`os.environ` on another, without unpredictable results. + See comments in https://github.com/gitpython-developers/GitPython/pull/1650. + """ + # CREATE_NEW_PROCESS_GROUP is needed for some ways of killing it afterwards. See: + # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal + # https://docs.python.org/3/library/subprocess.html#subprocess.CREATE_NEW_PROCESS_GROUP + creationflags = subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP + + # When using a shell, the shell is the direct subprocess, so the variable must be + # set in its environment, to affect its search behavior. (The "1" can be any value.) + if shell: + # The original may be immutable or reused by the caller. Make changes in a copy. + env = {} if env is None else dict(env) + env["NoDefaultCurrentDirectoryInExePath"] = "1" + + # When not using a shell, the current process does the search in a CreateProcessW + # API call, so the variable must be set in our environment. With a shell, this is + # unnecessary, in versions where https://github.com/python/cpython/issues/101283 is + # patched. If that is unpatched, then in the rare case the ComSpec environment + # variable is unset, the search for the shell itself is unsafe. Setting + # NoDefaultCurrentDirectoryInExePath in all cases, as is done here, is simpler and + # protects against that. (As above, the "1" can be any value.) + with patch_env("NoDefaultCurrentDirectoryInExePath", "1"): + return Popen( + command, + shell=shell, + env=env, + creationflags=creationflags, + **kwargs, + ) -if os.name == "nt": safer_popen = _safer_popen_windows else: safer_popen = Popen From 4191f7d598206ee829376e1a2a4e95ea571de500 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Mar 2024 14:29:48 -0500 Subject: [PATCH 0761/1392] Refactor kill_after_timeout logic so mypy can check it This changes how Git.execute defines the kill_process callback and how it performs checks, fixing two mypy errors on Windows about how the signal module doesn't have SIGKILL. In doing so, it also eliminates the need for the assertion added for safety and clarity in 2f017ac (#1761), since now kill_process is only defined if it is to be used (which is also guarded by a platform check, needed by mypy). As in dc95a76 before this, part of the change here is to replace some os.named-based checks with sys.platform-based checks, which is safe because, when one is specifically checking only for the distinction between native Windows and all other systems, one can use either approach. (See dc95a76 for more details on that.) --- git/cmd.py | 67 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 04037c06f..0adc8cf35 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1134,7 +1134,7 @@ def execute( if inline_env is not None: env.update(inline_env) - if os.name == "nt": + if sys.platform == "win32": cmd_not_found_exception = OSError if kill_after_timeout is not None: raise GitCommandError( @@ -1179,35 +1179,38 @@ def execute( if as_process: return self.AutoInterrupt(proc, command) - def kill_process(pid: int) -> None: - """Callback to kill a process.""" - if os.name == "nt": - raise AssertionError("Bug: This callback would be ineffective and unsafe on Windows, stopping.") - p = Popen(["ps", "--ppid", str(pid)], stdout=PIPE) - child_pids = [] - if p.stdout is not None: - for line in p.stdout: - if len(line.split()) > 0: - local_pid = (line.split())[0] - if local_pid.isdigit(): - child_pids.append(int(local_pid)) - try: - os.kill(pid, signal.SIGKILL) - for child_pid in child_pids: - try: - os.kill(child_pid, signal.SIGKILL) - except OSError: - pass - kill_check.set() # Tell the main routine that the process was killed. - except OSError: - # It is possible that the process gets completed in the duration after - # timeout happens and before we try to kill the process. - pass - return - - # END kill_process - - if kill_after_timeout is not None: + if sys.platform != "win32" and kill_after_timeout is not None: + + def kill_process(pid: int) -> None: + """Callback to kill a process. + + This callback implementation would be ineffective and unsafe on Windows. + """ + p = Popen(["ps", "--ppid", str(pid)], stdout=PIPE) + child_pids = [] + if p.stdout is not None: + for line in p.stdout: + if len(line.split()) > 0: + local_pid = (line.split())[0] + if local_pid.isdigit(): + child_pids.append(int(local_pid)) + try: + os.kill(pid, signal.SIGKILL) + for child_pid in child_pids: + try: + os.kill(child_pid, signal.SIGKILL) + except OSError: + pass + # Tell the main routine that the process was killed. + kill_check.set() + except OSError: + # It is possible that the process gets completed in the duration + # after timeout happens and before we try to kill the process. + pass + return + + # END kill_process + kill_check = threading.Event() watchdog = threading.Timer(kill_after_timeout, kill_process, args=(proc.pid,)) @@ -1218,10 +1221,10 @@ def kill_process(pid: int) -> None: newline = "\n" if universal_newlines else b"\n" try: if output_stream is None: - if kill_after_timeout is not None: + if sys.platform != "win32" and kill_after_timeout is not None: watchdog.start() stdout_value, stderr_value = proc.communicate() - if kill_after_timeout is not None: + if sys.platform != "win32" and kill_after_timeout is not None: watchdog.cancel() if kill_check.is_set(): stderr_value = 'Timeout: the command "%s" did not complete in %d ' "secs." % ( From 1ef336514a3513c7723a85ea295202560e7f40f0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Mar 2024 15:36:28 -0500 Subject: [PATCH 0762/1392] Factor communicate and watchdog logic to helper The goal is to improve readability by not repeating the platform and kill_after_timeout check three times. --- git/cmd.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 0adc8cf35..2ce690879 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1135,12 +1135,12 @@ def execute( env.update(inline_env) if sys.platform == "win32": - cmd_not_found_exception = OSError if kill_after_timeout is not None: raise GitCommandError( redacted_command, '"kill_after_timeout" feature is not supported on Windows.', ) + cmd_not_found_exception = OSError else: cmd_not_found_exception = FileNotFoundError # END handle @@ -1209,10 +1209,25 @@ def kill_process(pid: int) -> None: pass return - # END kill_process + def communicate() -> Tuple[AnyStr, AnyStr]: + watchdog.start() + out, err = proc.communicate() + watchdog.cancel() + if kill_check.is_set(): + err = 'Timeout: the command "%s" did not complete in %d ' "secs." % ( + " ".join(redacted_command), + kill_after_timeout, + ) + if not universal_newlines: + err = err.encode(defenc) + return out, err + + # END helpers kill_check = threading.Event() watchdog = threading.Timer(kill_after_timeout, kill_process, args=(proc.pid,)) + else: + communicate = proc.communicate # Wait for the process to return. status = 0 @@ -1221,18 +1236,7 @@ def kill_process(pid: int) -> None: newline = "\n" if universal_newlines else b"\n" try: if output_stream is None: - if sys.platform != "win32" and kill_after_timeout is not None: - watchdog.start() - stdout_value, stderr_value = proc.communicate() - if sys.platform != "win32" and kill_after_timeout is not None: - watchdog.cancel() - if kill_check.is_set(): - stderr_value = 'Timeout: the command "%s" did not complete in %d ' "secs." % ( - " ".join(redacted_command), - kill_after_timeout, - ) - if not universal_newlines: - stderr_value = stderr_value.encode(defenc) + stdout_value, stderr_value = communicate() # Strip trailing "\n". if stdout_value.endswith(newline) and strip_newline_in_stdout: # type: ignore stdout_value = stdout_value[:-1] From 4083dd896ad53f84e96f447b58cdf9acc1b02db3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Mar 2024 16:14:39 -0500 Subject: [PATCH 0763/1392] Fix new mypy confusion about kill_after_timeout type The refactoring in 1ef3365 before this added a new mypy error on non-Windows platforms, where mypy failed to infer that the type of kill_after_timeout was `float` (rather than `float | None`) at the point in the code where it was used as a captured variable in the newly introduced communicate() helper. This was even though the variable is never rebound there or in the enclosing scope that introduced it. So introducing and using a new variable that holds the same reference, which is sufficient to fix the problem and is the approach taken here, is not a behavioral change. --- git/cmd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 2ce690879..9fd2c532d 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1180,6 +1180,8 @@ def execute( return self.AutoInterrupt(proc, command) if sys.platform != "win32" and kill_after_timeout is not None: + # Help mypy figure out this is not None even when used inside communicate(). + timeout = kill_after_timeout def kill_process(pid: int) -> None: """Callback to kill a process. @@ -1216,7 +1218,7 @@ def communicate() -> Tuple[AnyStr, AnyStr]: if kill_check.is_set(): err = 'Timeout: the command "%s" did not complete in %d ' "secs." % ( " ".join(redacted_command), - kill_after_timeout, + timeout, ) if not universal_newlines: err = err.encode(defenc) @@ -1225,7 +1227,7 @@ def communicate() -> Tuple[AnyStr, AnyStr]: # END helpers kill_check = threading.Event() - watchdog = threading.Timer(kill_after_timeout, kill_process, args=(proc.pid,)) + watchdog = threading.Timer(timeout, kill_process, args=(proc.pid,)) else: communicate = proc.communicate From 3aeef466dea47d088fc7356b87589ee550c2030a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Mar 2024 17:24:56 -0500 Subject: [PATCH 0764/1392] Fix how Diffable annotates expected repo attribute The Diffable class's essential method, diff, requires self to have a repo attribute, which is outside Diffable's own responsibility to provide. This was already documented in the Diffable docstring, but the technique used to prevent mypy from treating accesses to that attribute as errors had several disadvantages: - Annotating `self.repo` is ambiguous, since typically *names* are annotated. Although mypy treated this afterwards, even in code appearing outside the implementation of Diffable.diff, to mean that Diffable instances have repo attributes, it is not clear how other type checkers, or possibly future version of mypy, would or should interpret this. It is also not obvious from reading the code that it should have an effect on static analysis of code outside the diff method body, but this can be verified by temporarily placing this code (unindented) after, and outside, the Diffable class: from git.types import Has_Repo def f(x: Has_Repo) -> None: pass f(Diffable()) This produces no new type errors from mypy, but if the annotated conditional attribute rebinding statement is commented out, then one of the type errors is that `f(Diffable())` passes an argument of the wrong type to `f`. - The annotation was on an attribute binding operation (assigning to `self.repo`), which mypy accurately reported as an error because the Diffable class had no `repo` instance attribute indicated in the code elsewhere and is a slotted class whose instances have no instance dictionaries to allow that actual binding operation to succeed. In addition to being an incorrect annotation, this had the effect of decreasing the number of related mypy errors only to one, instead of the hoped-for zero. - The rebinding of self.repo to itself was written in way that, if the statement were executed, rebinding would actually occur when the instance has a repo attribute. (See below for why this does not actually happen.) But this depends on the presence of a repo attribute to read from and then write to, so any change that allows mypy to infer that it is okay should also be enough to avoid the original mypy errors at issue in the first place. - If the statement were executed, it would fail at runtime if the instance had a repo attribute that were read-only. But there is no reason a subclass or sibling class used to provide the repo attribute should be unable to do this, such as by defining repo as a property with only a getter. - The condition that guarded the self.repo (re)binding operation was unintentionally incorrect in a way that caused it to be entirely unrelated to the git.typing.Has_Repo protocol it tried to refer to. The check was `hasattr(self, "Has_Repo")`, but the goal is not to check for an attribute named `Has_Repo`, but to check for an attribute named `repo`. With `Has_Repo` imported, the different condition `isinstance(self, Has_Repo)` would do that (the protocol is runtime-checkable). Or the condition `hasattr(self, "repo")` could be used. Instead, it worked the same as under the change: - if hasattr(self, "Has_Repo"): + if hasattr(self, "Any_Other_Nonexistent_Attribute"): The way it avoided a runtime error (and provably so, to mypy) was that it was always False, and mypy simply treated the annotation as usable information on both branches. This commit removes that code and instead adds a class-level annotation for a `repo` instance attribute, with no assignments. That eliminates the mypy error for the attempt to conditionally annotate `self.repo`, while also keeping the self.repo accesses from becoming static type errors again. As before, mypy treats Diffable as a static subclass of Has_Repo, but if an instance `x` of Diffable lacked a `repo` attribute (which would lead to a runtime error, as before, in realistic use, when `diff` were called on it), then `isinstance(x, Has_Repo)` will evaluate to `False`. Because the repo member of Has_Repo is not written as a method, it is a runtime error to use issubclass to check if a class is a subclass of it (even though isinstance does work), so adding this annotation to the Diffable class does not affect that either. Some behavior of Has_Repo may not be as expected, because of the different semantics of (a) the static type system checked by mypy and other static type checkers and (b) the actual dynamic type system of Python enforced at runtime by the implementation. Furthermore, this protocol is not used anywhere in GitPython, and since this commit, there is no attempt to use or reference it. Because it is public in the public git.types module, it should not be immediately removed, but it may make sense to deprecate it. However, this commit does not make that change. --- git/diff.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/git/diff.py b/git/diff.py index 966b73154..16e6be151 100644 --- a/git/diff.py +++ b/git/diff.py @@ -84,13 +84,16 @@ class Diffable: compatible type. :note: - Subclasses require a repo member as it is the case for - :class:`~git.objects.base.Object` instances, for practical reasons we do not + Subclasses require a repo member, as it is the case for + :class:`~git.objects.base.Object` instances. For practical reasons we do not derive from :class:`~git.objects.base.Object`. """ __slots__ = () + repo: "Repo" + """Repository to operate on. Must be provided by subclass or sibling class.""" + class Index: """Stand-in indicating you want to diff against the index.""" @@ -169,9 +172,6 @@ def diff( if paths is not None and not isinstance(paths, (tuple, list)): paths = [paths] - if hasattr(self, "Has_Repo"): - self.repo: "Repo" = self.repo - diff_cmd = self.repo.git.diff if other is self.Index: args.insert(0, "--cached") From f1cc1fe7c9497c97716af4c3071effaa37b89e79 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Mar 2024 20:17:55 -0500 Subject: [PATCH 0765/1392] Fix how HEAD annotates inherited commit property HEAD inherits its commit attribute from SymbolicReference, which defines it as a property using an explicit function call to `property`, rather than using it as a decorator. Due at least to https://github.com/python/mypy/issues/16827, mypy has trouble with this. Currently that assignment in SymbolicReference is marked as type[ignore], which suppresses a type error (or something mypy regards as one) in the call itself. Even without that type comment, the type of the SymboilicReference instance attribute produced by the property is inferred as Any. This commit does not change that. This commit replaces the HEAD class's partially successful attempt to annotate `commit` as `self.commit` in `__init__` with a fully recognized annotation for `commit` in the class scope (which is interpreted as an instance attribute). Merely removing the annotation in `__init__` is sufficient to make the mypy error for it go away, but this causes the inferred type of `x.commit` when `x` is a `HEAD` instance to be inferred as Any rather than the desired specific type Commit. That is why this also adds an annotation to the class to achieve that without causing its own mypy error as the old one did. Once the SymbolicReference.commit property has a more specific static type, however that is to be achieved, there will be no need for the annotation added in the HEAD class body here. (This differs from the situation in 3aeef46 before this, where Diffable does not inherit any `repo` attribute--and is intended not to--and therefore had to introduce an annotation for it, for mypy to accept it.) --- git/refs/head.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/git/refs/head.py b/git/refs/head.py index d546cb4ef..4da614abd 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -44,11 +44,13 @@ class HEAD(SymbolicReference): __slots__ = () + # TODO: This can be removed once SymbolicReference.commit has static type hints. + commit: "Commit" + def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME): if path != self._HEAD_NAME: raise ValueError("HEAD instance must point to %r, got %r" % (self._HEAD_NAME, path)) super().__init__(repo, path) - self.commit: "Commit" def orig_head(self) -> SymbolicReference: """ From e133018954cc38c09206de0236a281385f98e072 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Mar 2024 21:14:09 -0500 Subject: [PATCH 0766/1392] Broaden cygpath parameter annotation To fix a mypy error in Repo.__init__ where the epath variable, which can sometimes be a os.PathLike[str], does not match the str annotation for cygpath's path parameter, even though cygwin immediately calls str on that parameter. Pulling most of cygpath out into a new helper function is to help mypy correctly infer that the type of path is still compatible with the return type of str, when it is used in the return statement. I'm not sure this change is really the best solution at this time, because while it fixes the mypy error (without creating a new one), and cygpath is not listed in __all__, the docstring of advises to call Git.polish_url instead of cygpath. That method is public, annotates its parameter as str, and should not have its annotation broadened without considernig if it makes sense to do so or if its docstring needs to be updated. So either the cygpath docstring should be updated or this change should be undone and a different approach used to fix the type error in Repo.__init__. --- git/util.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/git/util.py b/git/util.py index 52f9dbdad..c07338582 100644 --- a/git/util.py +++ b/git/util.py @@ -404,9 +404,7 @@ def _cygexpath(drive: Optional[str], path: str) -> str: ) -def cygpath(path: str) -> str: - """Use :meth:`git.cmd.Git.polish_url` instead, that works on any environment.""" - path = str(path) # Ensure is str and not AnyPath. +def _cygpath(path: str) -> str: # Fix to use Paths when 3.5 dropped. Or to be just str if only for URLs? if not path.startswith(("/cygdrive", "//", "/proc/cygdrive")): for regex, parser, recurse in _cygpath_parsers: @@ -422,6 +420,11 @@ def cygpath(path: str) -> str: return path +def cygpath(path: PathLike) -> str: + """Use :meth:`git.cmd.Git.polish_url` instead, that works on any environment.""" + return _cygpath(str(path)) # Ensure is str and not AnyPath. + + _decygpath_regex = re.compile(r"(?:/proc)?/cygdrive/(\w)(/.*)?") From c34a466bb3c6b74ee2335bf8cb6511b9b17bdce5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Mar 2024 21:41:13 -0500 Subject: [PATCH 0767/1392] Have Repo.__init__ convert epath to str first instead In e133018 before this, I had noticed that Repo.__init__ implicitly relied on the implementation detail of cygpath that it converts its argument to str immediately. That detail was not reflected in the parameter type annotation, which could've been broader, so I had fixed the mypy error in Repo.__init__ by broadening cygpath's parameter type annotation. This undoes that change, putting the cygpath annotations back as before, and fixes the mypy error in another way, by having Repo.__init__ pass `str(epath)` instead of `epath` to cygpath. The reason is the concern noted in the e133018 commit message, that broadening the annotation made the way cygpath documents its relationshp to Git.polish_url no longer correct, and that it is not clear that the str(path) step in cygpath really ought to be made more than an implementation detail (which if done may require some documentation to be changed). Instances of str are usually direct instances (rather than of a subclass of str), in which case an extra call to str will (at least in CPython, and probably in all implementations) just return the same str object that was passed in. The performance penalty of this extra call to str should therefore be extremely small. --- git/repo/base.py | 2 +- git/util.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index a54591746..39a80b449 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -218,7 +218,7 @@ def __init__( # Given how the tests are written, this seems more likely to catch Cygwin # git used from Windows than Windows git used from Cygwin. Therefore # changing to Cygwin-style paths is the relevant operation. - epath = cygpath(epath) + epath = cygpath(str(epath)) epath = epath or path or os.getcwd() if not isinstance(epath, str): diff --git a/git/util.py b/git/util.py index c07338582..52f9dbdad 100644 --- a/git/util.py +++ b/git/util.py @@ -404,7 +404,9 @@ def _cygexpath(drive: Optional[str], path: str) -> str: ) -def _cygpath(path: str) -> str: +def cygpath(path: str) -> str: + """Use :meth:`git.cmd.Git.polish_url` instead, that works on any environment.""" + path = str(path) # Ensure is str and not AnyPath. # Fix to use Paths when 3.5 dropped. Or to be just str if only for URLs? if not path.startswith(("/cygdrive", "//", "/proc/cygdrive")): for regex, parser, recurse in _cygpath_parsers: @@ -420,11 +422,6 @@ def _cygpath(path: str) -> str: return path -def cygpath(path: PathLike) -> str: - """Use :meth:`git.cmd.Git.polish_url` instead, that works on any environment.""" - return _cygpath(str(path)) # Ensure is str and not AnyPath. - - _decygpath_regex = re.compile(r"(?:/proc)?/cygdrive/(\w)(/.*)?") From 4dfd48098c9c2ecf60ff29102c7c211311f8e93a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 Mar 2024 22:04:12 -0500 Subject: [PATCH 0768/1392] Fix how Remote annotates dynamic config-backed url attribute This fixes a mypy error by replacing an annotation on `self.url`, which is a mypy error, by adding an annotation at the class level for a `url` instance attribute. This adds a corresponding brief docstring for it as well, which may slightly improve usability, but the main impact is one less mypy error. --- git/remote.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/git/remote.py b/git/remote.py index 2c671582b..2bd674199 100644 --- a/git/remote.py +++ b/git/remote.py @@ -559,6 +559,9 @@ class Remote(LazyMixin, IterableObj): "--exec", ] + url: str # Obtained dynamically from _config_reader. See __getattr__ below. + """The URL configured for the remote.""" + def __init__(self, repo: "Repo", name: str) -> None: """Initialize a remote instance. @@ -570,7 +573,6 @@ def __init__(self, repo: "Repo", name: str) -> None: """ self.repo = repo self.name = name - self.url: str def __getattr__(self, attr: str) -> Any: """Allows to call this instance like ``remote.special(*args, **kwargs)`` to From e4fd2e343b4e7ece9df1269da9802b965ede4c51 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 8 Mar 2024 09:23:37 -0500 Subject: [PATCH 0769/1392] Drop wrong variable annotations in BlobFilter.__call__ This fixes two mypy errors by removing the annotations of the filter_parts and blob_parts local variables as lists. They only need to be sequences (more precisely, they only need to be sized and iterable, they don't even need to support subscripting in the way __call__ uses them), and mypy is able to infer their type. --- git/index/typ.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/index/typ.py b/git/index/typ.py index a7d2ad47a..c247fab99 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -12,7 +12,7 @@ # typing ---------------------------------------------------------------------- -from typing import NamedTuple, Sequence, TYPE_CHECKING, Tuple, Union, cast, List +from typing import NamedTuple, Sequence, TYPE_CHECKING, Tuple, Union, cast from git.types import PathLike @@ -60,8 +60,8 @@ def __call__(self, stage_blob: Tuple[StageType, Blob]) -> bool: path: Path = pathlike if isinstance(pathlike, Path) else Path(pathlike) # TODO: Change to use `PosixPath.is_relative_to` once Python 3.8 is no # longer supported. - filter_parts: List[str] = path.parts - blob_parts: List[str] = blob_path.parts + filter_parts = path.parts + blob_parts = blob_path.parts if len(filter_parts) > len(blob_parts): continue if all(i == j for i, j in zip(filter_parts, blob_parts)): From 94344b49b6dc38067cbb9aa2063043ac8e0a37eb Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 8 Mar 2024 11:05:28 -0500 Subject: [PATCH 0770/1392] Clarify CallableProgress vs. CallableRemoteProgress This corrects an overstatement in the git.types.CallableProgress docstring, which was introduced recently in 9e47083, and in which I had erroneously claimed that it was the most general type of object passed as a progress reporter for cloning. There are some non-None non-callable types that can also be passed and that are not encompassed by git.types.CallableProgress, somewhat confusingly including CallableRemoteProgress (which like RemoteProgress is not callable; rather, it wraps a callable and forwards progress information to it). In addition, None can be passed, and while git.types.CallableProgress does encompass it, it is not callable. (This is minor by comparison and I just added a brief note for it.) This also further expands the git.types.CallableProgress docstring, as well as the git.util.CallableRemoteProgress docstring, to clarify the distinction between them, as well as what "Callable" really signifies for CallableRemoteProgress (that it wraps and forwards to a callable, rather than itself being callable). --- git/types.py | 16 +++++++++++++++- git/util.py | 9 ++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/git/types.py b/git/types.py index 7e1cceaa8..8f71d079a 100644 --- a/git/types.py +++ b/git/types.py @@ -115,11 +115,25 @@ # Progress parameter type alias ----------------------------------------- CallableProgress = Optional[Callable[[int, Union[str, float], Union[str, float, None], str], None]] -"""General type of a progress reporter for cloning. +"""General type of a function or other callable used as a progress reporter for cloning. This is the type of a function or other callable that reports the progress of a clone, when passed as a ``progress`` argument to :meth:`Repo.clone ` or :meth:`Repo.clone_from `. + +:note: + Those :meth:`~git.repo.base.Repo.clone` and :meth:`~git.repo.base.Repo.clone_from` + methods also accept :meth:`~git.util.RemoteProgress` instances, including instances + of its :meth:`~git.util.CallableRemoteProgress` subclass. + +:note: + Unlike objects that match this type, :meth:`~git.util.RemoteProgress` instances are + not directly callable, not even when they are instances of + :meth:`~git.util.CallableRemoteProgress`, which wraps a callable and forwards + information to it but is not itself callable. + +:note: + This type also allows ``None``, for cloning without reporting progress. """ # ----------------------------------------------------------------------------------- diff --git a/git/util.py b/git/util.py index 52f9dbdad..880d6625a 100644 --- a/git/util.py +++ b/git/util.py @@ -749,7 +749,14 @@ def update( class CallableRemoteProgress(RemoteProgress): - """An implementation forwarding updates to any callable.""" + """A :class:`RemoteProgress` implementation forwarding updates to any callable. + + :note: + Like direct instances of :class:`RemoteProgress`, instances of this + :class:`CallableRemoteProgress` class are not themselves directly callable. + Rather, instances of this class wrap a callable and forward to it. This should + therefore not be confused with :class:`git.types.CallableProgress`. + """ __slots__ = ("_callable",) From 8e8b87a573dd7ca27d3556788c0da20faa803cfd Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 8 Mar 2024 14:06:37 -0500 Subject: [PATCH 0771/1392] Fix RootModule.update `ignore[override]` suppression It was on the `previous_commit` parameter, and ineffective. It may be that the original intent of the suppression was to suppress only incompatible override type errors due to the addition of that parameter, but there are other paramters that also mismatch by having a different name or by being absent even though present in the overridden base-class method. Furthermore, even coresponding parameters mismatch in position due to the insertion of the `previous_commit` parameter, so even if there is some way to do a more fine-grained suppression than applying `ignore[override]` to the entire method signature, that would be insufficient here. This fixes one mypy error. It does so by causing it to be suppressed effectively, not by fixing the underlying issue, which may not be fixable since it would entail a breaking change to fix. However, this does not introduce any new suppressions, just fixes an existing suppression so it is effective (and probably so it operates as intended, though maybe it is slightly stronger than intended as discussed above). --- git/objects/submodule/root.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index ac0f7ad94..d0d1818a9 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -75,9 +75,9 @@ def _clear_cache(self) -> None: # { Interface - def update( + def update( # type: ignore[override] self, - previous_commit: Union[Commit_ish, None] = None, # type: ignore[override] + previous_commit: Union[Commit_ish, None] = None, recursive: bool = True, force_remove: bool = False, init: bool = True, From 938f2725eb137492a459b4210ae58565e26e4fd5 Mon Sep 17 00:00:00 2001 From: jcole-crowdstrike Date: Tue, 5 Mar 2024 07:37:36 -0800 Subject: [PATCH 0772/1392] Setting universal_newlines=False explicitly in fetch() and pull(). This fix avoids encoding problems on Windows, as was done for fetch() in the previous commit. However, here it is being made more explicit and adds the same fix for the pull function. --- git/remote.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/remote.py b/git/remote.py index 6cb33bede..bbe1aea97 100644 --- a/git/remote.py +++ b/git/remote.py @@ -1068,7 +1068,7 @@ def fetch( Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_fetch_options) proc = self.repo.git.fetch( - "--", self, *args, as_process=True, with_stdout=False, v=verbose, **kwargs + "--", self, *args, as_process=True, with_stdout=False, universal_newlines=False, v=verbose, **kwargs ) res = self._get_fetch_info_from_stderr(proc, progress, kill_after_timeout=kill_after_timeout) if hasattr(self.repo.odb, "update_cache"): @@ -1122,7 +1122,7 @@ def pull( Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_pull_options) proc = self.repo.git.pull( - "--", self, refspec, with_stdout=False, as_process=True, universal_newlines=True, v=True, **kwargs + "--", self, refspec, with_stdout=False, as_process=True, universal_newlines=False, v=True, **kwargs ) res = self._get_fetch_info_from_stderr(proc, progress, kill_after_timeout=kill_after_timeout) if hasattr(self.repo.odb, "update_cache"): From 8b8c76a707155e32e7538136053ff00c3231cc92 Mon Sep 17 00:00:00 2001 From: jcole-crowdstrike Date: Fri, 8 Mar 2024 12:45:33 -0800 Subject: [PATCH 0773/1392] Fixing stripping issue causing passing tests to be interpreted as failures Removing this block of code that strips away nonprintable ASCII characters, as it is causing some tests to fail inappropriately. Successful tests are identified by the suffix ", done", which is being removed from some messages due to this aforementioned block of code. The codeblock probably intends to just strip away traililng non-ASCII characters from strings, but does not break, and so will instead continue to iterate through the string and then strip away both printable/nonprintable ASCII; this is causing the loss of the ", done" because anytime a nonprintable character is anywhere but the end of the string then the entire section of string after the character is truncated. Looking back through commits, this codeblock was added in 882ebb153e14488b275e374ccebcdda1dea22dd7 as a workaround after the addition of universal_newlines. Seeing as we have removed universal_newlines in the previous commit, we can also remove this codeblock. In future, if something likw this is needed, then maybe a join that concatenates all printable ASCII characters would serve better (e.g., "".join(b for b in line if b >= 32)), instead of trying to cut out the nonprintable chars. --- git/util.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/git/util.py b/git/util.py index 52f9dbdad..63c9b1e99 100644 --- a/git/util.py +++ b/git/util.py @@ -611,20 +611,6 @@ def _parse_progress_line(self, line: AnyStr) -> None: self.error_lines.append(self._cur_line) return - # Find escape characters and cut them away - regex will not work with - # them as they are non-ASCII. As git might expect a tty, it will send them. - last_valid_index = None - for i, c in enumerate(reversed(line_str)): - if ord(c) < 32: - # its a slice index - last_valid_index = -i - 1 - # END character was non-ASCII - # END for each character in line - if last_valid_index is not None: - line_str = line_str[:last_valid_index] - # END cut away invalid part - line_str = line_str.rstrip() - cur_count, max_count = None, None match = self.re_op_relative.match(line_str) if match is None: From 1cdec7afcd8e4d69dbb9f8e596282c6ddb6414be Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 8 Mar 2024 17:12:28 -0500 Subject: [PATCH 0774/1392] Fix wrong class name in git.objects.tag docstring In 6126997 (#1850), I had meant to have the git.objects.tag module docstring say that the module defined the TagObject class, to help ensure no one would confuse this with the TagReference class. But I instead had it wrongly say it defined the TagReference class! This fixes that. --- git/objects/tag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/tag.py b/git/objects/tag.py index 49fb97e37..fd3cdd630 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -5,7 +5,7 @@ """Provides an :class:`~git.objects.base.Object`-based type for annotated tags. -This defines the :class:`TagReference` class, which represents annotated tags. +This defines the :class:`TagObject` class, which represents annotated tags. For lightweight tags, see the :mod:`git.refs.tag` module. """ From ed6ead969914597a56aca752aadd044a276f1137 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 8 Mar 2024 17:37:00 -0500 Subject: [PATCH 0775/1392] Correct and clarify Diffable.diff docstring In cd16a35 (#1725), I had taken "Treeish" to mean the type of that exact name, git.index.base.Treeish. But that type is only used within the git.index package (actually only in git.index.base itself). It is also nonpublic: git.index.base.__all__ exists and does not list it. So most likely this was not intended in the git.diff.Diffable.diff docstring. Even if intended, it does not appear accurate, since the git.index.base.Treeish union includes bytes, and the logic in Diffable.diff and its helpers does not appear to accommodate bytes. A closer type is the public git.types.Tree_ish union, which is narrower than git.index.base.Treeish, including neither str nor bytes. However, it does not include str, and Diffable.diff does accept str to specify a tree-ish for diff-ing. It may be that "Treeish" in the pre-#1725 docstring was capitalized for some reason other than to identify a type defined in GitPython's code. For now, I've changed it to refer to git.types.Tree_ish, but also explicitly documented that a string can be used to specify a tree-ish -- which is independently valuable, since previously the effect of passing a str instance to the diff method was not stated anywhere in the method docstring. To clarify further, I included a link to tree-ish in gitglossary(7) as well. In addition, the original wording before cd16a35 had included "(type)", which I had erroneously assumed was just meant to state that it is a type (i.e. a class), so I had wrongly removed it without replacing it with anything when making it into a reference to a type. But it was really an attempt to clarify that Diffable.Index should be used directly, rather than an instance of it. That is in effect the opposite of merely pointing out that it is a class; it is to express that it should be used in a way that does not depend in any way on it being a class. This commit has the docstring explicitly state that. --- git/diff.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/git/diff.py b/git/diff.py index 16e6be151..ec7fe7755 100644 --- a/git/diff.py +++ b/git/diff.py @@ -122,12 +122,21 @@ def diff( This the item to compare us with. * If ``None``, we will be compared to the working tree. - * If :class:`~git.index.base.Treeish`, it will be compared against the - respective tree. - * If :class:`Diffable.Index`, it will be compared against the index. - * If :attr:`git.NULL_TREE`, it will compare against the empty tree. - * It defaults to :class:`Diffable.Index` so that the method will not by - default fail on bare repositories. + + * If :class:`~git.types.Tree_ish`, it will be compared against the + respective tree. (See https://git-scm.com/docs/gitglossary#def_tree-ish.) + This can also be passed as a string. + + * If :class:`Diffable.Index`, it will be compared against the index. Use the + type object :class:`Index` itself, without attempting to instantiate it. + (That is, you should treat :class:`Index` as an opqaue constant. Don't + rely on it being a class or even callable.) + + * If :attr:`git.NULL_TREE `, it will compare against the empty + tree. + + This parameter defaults to :class:`Diffable.Index` (rather than ``None``) so + that the method will not by default fail on bare repositories. :param paths: This a list of paths or a single path to limit the diff to. It will only @@ -143,7 +152,7 @@ def diff( sides of the diff. :return: - :class:`DiffIndex` + A :class:`DiffIndex` representing the computed diff. :note: On a bare repository, `other` needs to be provided as From 0e1df29c171e8d6d7dd62f5840d88a9bfd0ef3ab Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 8 Mar 2024 23:27:58 -0500 Subject: [PATCH 0776/1392] Start fixing diff and _process_diff_args type annotations This fixes three mypy errors by modifying the Diffable.diff, Diffable._process_diff_args, and its IndexFile._process_diff_args override. This change, so far, adds no suppressions, and even removes some preexisting suppressions that were ineffective because of how they were written (not being on the first line of the def). However, this is not a complete fix of those annotations themselves. This is because although the `other` parameter to Diffable.diff and Diffable._process_diff_args does not appear intended to have been as broad as including object in the union had the effect of making it -- any union with object as an alternative is equivalent to object itself -- it should nonetheless be broader than the changes here make it. Once that is fixed, it may not be possible to maintain compliance with the Liskov substitution principle, in which case a suppression, corresponding to one of those that was removed but fixed so it has an effect, may need to be introduced, since actually fixing the LSP violation would be a breaking change. Specifically, as seen in 797e962 and 09053c5 in #1285, the object alternative was originally intended not to indicate that any object is allowed, but instead to allow the NULL_TREE constant, which was (and currently remains) implemented as a direct instance of object. The type of NULL_TREE is not the core problem. It can be fixed to allow a narrowly scoped annotation. One way is by making NULL_TREE an enumeration constant and annotating `Literal[NULL_TREE]`. Rather, the problem is that the IndexFile.diff override really does not accept NULL_TREE. It may not be feasible to modify it to accept it. Even if that is to be done, it should be for runtime correctness, and likely have a test case added for it, and may not be suitable for inclusion alongside these static typing fixes. So when the base class method's annotation is broadened to add such a literal or equivalent, the override's annotation in the derived class may not reasonably be able to change accordingly, as LSP would dictate. Part of the change here adds a missing os.PathLike[str] alternative to _process_diff_args. For now I've opted to include both str (already present) and os.PathLike[str] separately, rather than consolidating them into PathLike (in GitPython, PathLike is git.types.PathLike, which is Union[str, os.PathLike[str]]). The reason is that, while some str arguments may be paths, others may be options. However, that stylistic choice may not be justified, since it is at odds with some existing uses of PathLike in GitPython to cover both str as a path and str as a non-path, including in the implementation of Diffable.diff itself. So those may be consolidated in a forthcoming commit. --- git/diff.py | 12 +++++++----- git/index/base.py | 8 ++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/git/diff.py b/git/diff.py index ec7fe7755..cf86e95fb 100644 --- a/git/diff.py +++ b/git/diff.py @@ -3,6 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import os import re from git.cmd import handle_process_output @@ -98,8 +99,9 @@ class Index: """Stand-in indicating you want to diff against the index.""" def _process_diff_args( - self, args: List[Union[str, "Diffable", Type["Diffable.Index"], object]] - ) -> List[Union[str, "Diffable", Type["Diffable.Index"], object]]: + self, + args: List[Union[str, os.PathLike[str], "Diffable", Type["Index"]]], + ) -> List[Union[str, os.PathLike[str], "Diffable", Type["Index"]]]: """ :return: Possibly altered version of the given args list. @@ -110,7 +112,7 @@ def _process_diff_args( def diff( self, - other: Union[Type["Index"], "Tree", "Commit", None, str, object] = Index, + other: Union[Type["Index"], "Tree", "Commit", str, None] = Index, paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, **kwargs: Any, @@ -159,7 +161,7 @@ def diff( :class:`~Diffable.Index`, or as :class:`~git.objects.tree.Tree` or :class:`~git.objects.commit.Commit`, or a git command error will occur. """ - args: List[Union[PathLike, Diffable, Type["Diffable.Index"], object]] = [] + args: List[Union[PathLike, Diffable, Type["Diffable.Index"]]] = [] args.append("--abbrev=40") # We need full shas. args.append("--full-index") # Get full index paths, not only filenames. @@ -195,7 +197,7 @@ def diff( args.insert(0, self) - # paths is a list here, or None. + # paths is a list or tuple here, or None. if paths: args.append("--") args.extend(paths) diff --git a/git/index/base.py b/git/index/base.py index 249144d54..6fb5abdcb 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -644,9 +644,9 @@ def write_tree(self) -> Tree: return root_tree def _process_diff_args( - self, # type: ignore[override] - args: List[Union[str, "git_diff.Diffable", Type["git_diff.Diffable.Index"]]], - ) -> List[Union[str, "git_diff.Diffable", Type["git_diff.Diffable.Index"]]]: + self, + args: List[Union[str, os.PathLike[str], "git_diff.Diffable", Type["git_diff.Diffable.Index"]]], + ) -> List[Union[str, os.PathLike[str], "git_diff.Diffable", Type["git_diff.Diffable.Index"]]]: try: args.pop(args.index(self)) except IndexError: @@ -1478,7 +1478,7 @@ def reset( # @ default_index, breaks typing for some reason, copied into function def diff( - self, # type: ignore[override] + self, other: Union[Type["git_diff.Diffable.Index"], "Tree", "Commit", str, None] = git_diff.Diffable.Index, paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, From 62c0823da472a4f9c714827291d15a6f36abebda Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 00:09:45 -0500 Subject: [PATCH 0777/1392] Consolidate str and os.PathLike[str] (use GitPython's PathLike) Where they had been introduced and written separately in 0e1df29 before this. The main reason is that it would have to be quoted as a string for compatibility with Python 3.8 and lower, since os.PathLike only defined __class_getitem__ in 3.8 and later. (The definition in git.types already does that.) In addition, the benefit of keeping them separate was negligible, as mentioned in 0e1df29. --- git/diff.py | 5 ++--- git/index/base.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/git/diff.py b/git/diff.py index cf86e95fb..04e38d209 100644 --- a/git/diff.py +++ b/git/diff.py @@ -3,7 +3,6 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -import os import re from git.cmd import handle_process_output @@ -100,8 +99,8 @@ class Index: def _process_diff_args( self, - args: List[Union[str, os.PathLike[str], "Diffable", Type["Index"]]], - ) -> List[Union[str, os.PathLike[str], "Diffable", Type["Index"]]]: + args: List[Union[PathLike, "Diffable", Type["Index"]]], + ) -> List[Union[PathLike, "Diffable", Type["Index"]]]: """ :return: Possibly altered version of the given args list. diff --git a/git/index/base.py b/git/index/base.py index 6fb5abdcb..357de6bd9 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -645,8 +645,8 @@ def write_tree(self) -> Tree: def _process_diff_args( self, - args: List[Union[str, os.PathLike[str], "git_diff.Diffable", Type["git_diff.Diffable.Index"]]], - ) -> List[Union[str, os.PathLike[str], "git_diff.Diffable", Type["git_diff.Diffable.Index"]]]: + args: List[Union[PathLike, "git_diff.Diffable", Type["git_diff.Diffable.Index"]]], + ) -> List[Union[PathLike, "git_diff.Diffable", Type["git_diff.Diffable.Index"]]]: try: args.pop(args.index(self)) except IndexError: From 7204cc106cc055b5bdd841dcc2a1c4c18e60fcf3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 10:34:59 -0500 Subject: [PATCH 0778/1392] Further clarify Diffable.diff docstring Along the lines of ed6ead9. --- git/diff.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/git/diff.py b/git/diff.py index 04e38d209..4f4cab048 100644 --- a/git/diff.py +++ b/git/diff.py @@ -157,8 +157,9 @@ def diff( :note: On a bare repository, `other` needs to be provided as - :class:`~Diffable.Index`, or as :class:`~git.objects.tree.Tree` or - :class:`~git.objects.commit.Commit`, or a git command error will occur. + :class:`~Diffable.Index`, or as an instance of + :class:`~git.objects.tree.Tree` or :class:`~git.objects.commit.Commit`, or a + git command error will occur. """ args: List[Union[PathLike, Diffable, Type["Diffable.Index"]]] = [] args.append("--abbrev=40") # We need full shas. From 2f5e258d2b232685993c589b51445e51b0c2a185 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 09:58:52 -0500 Subject: [PATCH 0779/1392] Annotate _process_diff_args without Diffable.Index This removes the static type of Diffable's nested Index class, Type["Index"], from the Diffable._process_diff_args method, and its override IndexFile._process_diff_args. Further changes related to this remain necessary, and at this time, this adds one mypy error. The _process_diff_args methods did not handle Diffable.Index. The base class method would pass it through, but it would not be usable when then passed to "git diff". Instead, it is handled with correct behavior at runtime in Diffable.diff and its IndexFile.diff override, which handle it themselves and ensure it is never passed to any _process_diff_args implementation. That was already the case. The change here is mostly to type hints, removing it from the _process_diff_args annotations, since those methods have never actually worked with it and it probably wouldn't make sense for them to handle it. However, this does also attempt to help mypy figure out that Diffable.Index cannot end up as an element of its local variable `args`. This attempt is unsuccessful. The problem with the `args` local variable may be the reason including Index in the parameter type annotations appeared correct or necessary before. The issue is that Diffable.Index, even though it ought to be used as an opaque constant, is a class. Its static type is Type[Diffable.Index], but mypy is unable to infer that there are no other objects of that static type, because if Diffable.Index were subclassed, and the type object produced from doing so (i.e. the subclass itself) were passed as an `other` argument to the diff method, then the `other is Diffable.Index` condition would evaluate to False. Therefore to solve this problem it should be sufficient to decorate the Diffable.Index class as `@final`. This works for pyright (and thus also pylance), but it does not work for mypy (even in 1.9.0). So that still has to be solved, and (other than via suppressions) it may be necessary to make Diffable.Index an enumeration constant rather than a class, so it can be annotated as a literal. --- git/diff.py | 11 ++++++----- git/index/base.py | 5 ++--- git/types.py | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/git/diff.py b/git/diff.py index 4f4cab048..91dd45704 100644 --- a/git/diff.py +++ b/git/diff.py @@ -28,7 +28,7 @@ TYPE_CHECKING, cast, ) -from git.types import PathLike, Literal +from git.types import Literal, PathLike, final if TYPE_CHECKING: from .objects.tree import Tree @@ -94,13 +94,14 @@ class Diffable: repo: "Repo" """Repository to operate on. Must be provided by subclass or sibling class.""" + @final class Index: """Stand-in indicating you want to diff against the index.""" def _process_diff_args( self, - args: List[Union[PathLike, "Diffable", Type["Index"]]], - ) -> List[Union[PathLike, "Diffable", Type["Index"]]]: + args: List[Union[PathLike, "Diffable"]], + ) -> List[Union[PathLike, "Diffable"]]: """ :return: Possibly altered version of the given args list. @@ -161,7 +162,7 @@ def diff( :class:`~git.objects.tree.Tree` or :class:`~git.objects.commit.Commit`, or a git command error will occur. """ - args: List[Union[PathLike, Diffable, Type["Diffable.Index"]]] = [] + args: List[Union[PathLike, Diffable]] = [] args.append("--abbrev=40") # We need full shas. args.append("--full-index") # Get full index paths, not only filenames. @@ -184,7 +185,7 @@ def diff( paths = [paths] diff_cmd = self.repo.git.diff - if other is self.Index: + if other is Diffable.Index: args.insert(0, "--cached") elif other is NULL_TREE: args.insert(0, "-r") # Recursive diff-tree. diff --git a/git/index/base.py b/git/index/base.py index 357de6bd9..591f8c6f6 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -645,8 +645,8 @@ def write_tree(self) -> Tree: def _process_diff_args( self, - args: List[Union[PathLike, "git_diff.Diffable", Type["git_diff.Diffable.Index"]]], - ) -> List[Union[PathLike, "git_diff.Diffable", Type["git_diff.Diffable.Index"]]]: + args: List[Union[PathLike, "git_diff.Diffable"]], + ) -> List[Union[PathLike, "git_diff.Diffable"]]: try: args.pop(args.index(self)) except IndexError: @@ -1494,7 +1494,6 @@ def diff( Will only work with indices that represent the default git index as they have not been initialized with a stream. """ - # Only run if we are the default repository index. if self._file_path != self._index_path(): raise AssertionError("Cannot call %r on indices that do not represent the default git index" % self.diff()) diff --git a/git/types.py b/git/types.py index 8f71d079a..652a73940 100644 --- a/git/types.py +++ b/git/types.py @@ -22,6 +22,7 @@ TypedDict, Protocol, SupportsIndex as SupportsIndex, + final, runtime_checkable, ) else: @@ -30,6 +31,7 @@ SupportsIndex as SupportsIndex, TypedDict, Protocol, + final, runtime_checkable, ) From 65863a28c2bfebe084e1c86d409fbf68b0b33a76 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 13:34:55 -0500 Subject: [PATCH 0780/1392] Make NULL_TREE and Index precisely annotatable This creates a git.diff.DiffConstants enumeration and makes the git.diff.NULL_TREE and git.diff.Diffable.Index objects constants in it. This allows them (including as an alternative in a union) to be annotated as literals: - Literal[DiffConstants.NULL_TREE] - Literal[DIffConstants.INDEX] Although the enumeration type must unfortunately be included in the annotations as shown above (at least mypy requires this), using the objects/values themselves does not require any code to change. So this shouldn't break anything at runtime for code using GitPython, unless it has relied on NULL_TREE being a direct instance of object, or relied on Diffable.Index (all useful uses of which were also as an opaque constant) being defined as a class. More specifically, all the ways that NULL_TREE and Index could be *accessed* before are still working, because: - NULL_TREE is aliased at module level, where it was before. It is also aliased at class level in Diffable, for consistency. - INDEX is aliased at class level in Diffable, as Index, where it was before. This way, Diffable.Index can still be used. It is also aliased, in the same scope, as INDEX. Because it is, and in effect has always been, a constant, the new INDEX spelling is preferable. But there is no major disadvantage of the old Index spelling, so in docstrings I have made no effort at this time to discourage its use. (If GitPython ever uses all-caps identifiers strictly as constants, then the clarity benefit of ensuring only the INDEX version is used will be greater, and then it might make sense to deprecate Index. However, this seems unlikely to happen as it would be a breaking change, due to the way functions like git.reset rebind Git.GIT_PYTHON_GIT_EXECUTABLE.) INDEX is also aliased at module level, for consistency. - NULL_TREE is still included in git.diff.__all__, causing it to be recognized as public in git.diff and also to be accessible as an attribute of the top-level git module (which currently uses wildcard imports). For consistency, I have also included INDEX in __all__. - Because there is a benefit to being able to freely referene the new DiffConstants enumeration in annotations (though in practice this may mostly be to implementers of new Diffable subclasses), I have also included DiffConstants in __all__. In addition, the name DiffConstants (rather than, e.g., Constants) should avoid confusion even if it ends up in another scope unexpectedly. To avoid a situation where users/developers may erroneously think these aliases are different from each other, I have documented the situation in the docstrings for each, referring to the others. (Sphinx does not automatically use the original docstring for an aliased name introduced in this way, and there is also arguably a clarity benefit to their differing wording, such as how each refers *only* to the others.) Other docstings are also updated. This commit completes the change begun in 2f5e258 before this, resolving the one mypy error it added. But this does not complete the larger change begun in 0e1df29: - One of the effects of this change is to make it possible to annotate precisely for NULL_TREE, either by using Literal[DiffConstants.NULL_TREE] or consolidating it with the Literal[DiffConstants.INDEX] alternative by including DiffConstants in the union instead of either/both of them. - But that is not yet done here, and when it is done for Diffable.diff, the LSP issue in IndexFile.diff, which does not currently accommodate NULL_TREE, will resurface (or, rather, be rightly revealed by mypy, in a way that is specifically clear). --- git/diff.py | 102 +++++++++++++++++++++++++++++++++++----------- git/index/base.py | 5 +-- git/types.py | 2 - 3 files changed, 81 insertions(+), 28 deletions(-) diff --git a/git/diff.py b/git/diff.py index 91dd45704..cebc1b3ed 100644 --- a/git/diff.py +++ b/git/diff.py @@ -3,6 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import enum import re from git.cmd import handle_process_output @@ -22,13 +23,12 @@ Match, Optional, Tuple, - Type, TypeVar, Union, TYPE_CHECKING, cast, ) -from git.types import Literal, PathLike, final +from git.types import Literal, PathLike if TYPE_CHECKING: from .objects.tree import Tree @@ -48,10 +48,55 @@ # ------------------------------------------------------------------------ -__all__ = ("Diffable", "DiffIndex", "Diff", "NULL_TREE") +__all__ = ("DiffConstants", "NULL_TREE", "INDEX", "Diffable", "DiffIndex", "Diff") -NULL_TREE = object() -"""Special object to compare against the empty tree in diffs.""" + +@enum.unique +class DiffConstants(enum.Enum): + """Special objects for :meth:`Diffable.diff`. + + See the :meth:`Diffable.diff` method's ``other`` parameter, which accepts various + values including these. + + :note: + These constants are also available as attributes of the :mod:`git.diff` module, + the :class:`Diffable` class and its subclasses and instances, and the top-level + :mod:`git` module. + """ + + NULL_TREE = enum.auto() + """Stand-in indicating you want to compare against the empty tree in diffs. + + Also accessible as :const:`git.NULL_TREE`, :const:`git.diff.NULL_TREE`, and + :const:`Diffable.NULL_TREE`. + """ + + INDEX = enum.auto() + """Stand-in indicating you want to diff against the index. + + Also accessible as :const:`git.INDEX`, :const:`git.diff.INDEX`, and + :const:`Diffable.INDEX`, as well as :const:`Diffable.Index`. The latter has been + kept for backward compatibility and made an alias of this, so it may still be used. + """ + + +NULL_TREE: Literal[DiffConstants.NULL_TREE] = DiffConstants.NULL_TREE +"""Stand-in indicating you want to compare against the empty tree in diffs. + +See :meth:`Diffable.diff`, which accepts this as a value of its ``other`` parameter. + +This is an alias of :const:`DiffConstants.NULL_TREE`, which may also be accessed as +:const:`git.NULL_TREE` and :const:`Diffable.NULL_TREE`. +""" + +INDEX: Literal[DiffConstants.INDEX] = DiffConstants.INDEX +"""Stand-in indicating you want to diff against the index. + +See :meth:`Diffable.diff`, which accepts this as a value of its ``other`` parameter. + +This is an alias of :const:`DiffConstants.INDEX`, which may also be accessed as +:const:`git.INDEX` and :const:`Diffable.INDEX`, as well as :const:`Diffable.Index`. +""" _octal_byte_re = re.compile(rb"\\([0-9]{3})") @@ -84,7 +129,7 @@ class Diffable: compatible type. :note: - Subclasses require a repo member, as it is the case for + Subclasses require a :attr:`repo` member, as it is the case for :class:`~git.objects.base.Object` instances. For practical reasons we do not derive from :class:`~git.objects.base.Object`. """ @@ -94,9 +139,25 @@ class Diffable: repo: "Repo" """Repository to operate on. Must be provided by subclass or sibling class.""" - @final - class Index: - """Stand-in indicating you want to diff against the index.""" + NULL_TREE = NULL_TREE + """Stand-in indicating you want to compare against the empty tree in diffs. + + See the :meth:`diff` method, which accepts this as a value of its ``other`` + parameter. + + This is the same as :const:`DiffConstants.NULL_TREE`, and may also be accessed as + :const:`git.NULL_TREE` and :const:`git.diff.NULL_TREE`. + """ + + INDEX = Index = INDEX + """Stand-in indicating you want to diff against the index. + + See the :meth:`diff` method, which accepts this as a value of its ``other`` + parameter. + + This is the same as :const:`DiffConstants.INDEX`, and may also be accessed as + :const:`git.INDEX` and :const:`git.diff.INDEX`. + """ def _process_diff_args( self, @@ -112,7 +173,7 @@ def _process_diff_args( def diff( self, - other: Union[Type["Index"], "Tree", "Commit", str, None] = Index, + other: Union[Literal[DiffConstants.INDEX], "Tree", "Commit", str, None] = INDEX, paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, **kwargs: Any, @@ -125,20 +186,15 @@ def diff( * If ``None``, we will be compared to the working tree. - * If :class:`~git.types.Tree_ish`, it will be compared against the - respective tree. (See https://git-scm.com/docs/gitglossary#def_tree-ish.) - This can also be passed as a string. + * If a :class:`~git.types.Tree_ish` or string, it will be compared against + the respective tree. - * If :class:`Diffable.Index`, it will be compared against the index. Use the - type object :class:`Index` itself, without attempting to instantiate it. - (That is, you should treat :class:`Index` as an opqaue constant. Don't - rely on it being a class or even callable.) + * If :const:`INDEX`, it will be compared against the index. - * If :attr:`git.NULL_TREE `, it will compare against the empty - tree. + * If :const:`NULL_TREE`, it will compare against the empty tree. - This parameter defaults to :class:`Diffable.Index` (rather than ``None``) so - that the method will not by default fail on bare repositories. + This parameter defaults to :const:`INDEX` (rather than ``None``) so that the + method will not by default fail on bare repositories. :param paths: This a list of paths or a single path to limit the diff to. It will only @@ -185,7 +241,7 @@ def diff( paths = [paths] diff_cmd = self.repo.git.diff - if other is Diffable.Index: + if other is INDEX: args.insert(0, "--cached") elif other is NULL_TREE: args.insert(0, "-r") # Recursive diff-tree. @@ -218,7 +274,7 @@ def diff( class DiffIndex(List[T_Diff]): - R"""An Index for diffs, allowing a list of :class:`Diff`\s to be queried by the diff + R"""An index for diffs, allowing a list of :class:`Diff`\s to be queried by the diff properties. The class improves the diff handling convenience. diff --git a/git/index/base.py b/git/index/base.py index 591f8c6f6..48e843822 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -76,11 +76,10 @@ Sequence, TYPE_CHECKING, Tuple, - Type, Union, ) -from git.types import Commit_ish, PathLike +from git.types import Commit_ish, Literal, PathLike if TYPE_CHECKING: from subprocess import Popen @@ -1479,7 +1478,7 @@ def reset( # @ default_index, breaks typing for some reason, copied into function def diff( self, - other: Union[Type["git_diff.Diffable.Index"], "Tree", "Commit", str, None] = git_diff.Diffable.Index, + other: Union[Literal[git_diff.DiffConstants.INDEX], "Tree", "Commit", str, None] = git_diff.INDEX, paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, **kwargs: Any, diff --git a/git/types.py b/git/types.py index 652a73940..8f71d079a 100644 --- a/git/types.py +++ b/git/types.py @@ -22,7 +22,6 @@ TypedDict, Protocol, SupportsIndex as SupportsIndex, - final, runtime_checkable, ) else: @@ -31,7 +30,6 @@ SupportsIndex as SupportsIndex, TypedDict, Protocol, - final, runtime_checkable, ) From c9952e14b51140e7307a031149fbe15ab5911abd Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 14:49:46 -0500 Subject: [PATCH 0781/1392] Fix Sphinx references; give Diffable.Index a docstring --- git/diff.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/git/diff.py b/git/diff.py index cebc1b3ed..e185b4cb3 100644 --- a/git/diff.py +++ b/git/diff.py @@ -149,14 +149,30 @@ class Diffable: :const:`git.NULL_TREE` and :const:`git.diff.NULL_TREE`. """ - INDEX = Index = INDEX + INDEX = INDEX """Stand-in indicating you want to diff against the index. See the :meth:`diff` method, which accepts this as a value of its ``other`` parameter. This is the same as :const:`DiffConstants.INDEX`, and may also be accessed as - :const:`git.INDEX` and :const:`git.diff.INDEX`. + :const:`git.INDEX` and :const:`git.diff.INDEX`, as well as :class:`Diffable.INDEX`, + which is kept for backward compatibility (it is now defined an alias of this). + """ + + Index = INDEX + """Stand-in indicating you want to diff against the index + (same as :const:`~Diffable.INDEX`). + + This is an alias of :const:`~Diffable.INDEX`, for backward compatibility. See + :const:`~Diffable.INDEX` and :meth:`diff` for details. + + :note: + Although always meant for use as an opaque constant, this was formerly defined + as a class. Its usage is unchanged, but static type annotations that attempt + to permit only this object must be changed to avoid new mypy errors. This was + previously not possible to do, though ``Type[Diffable.Index]`` approximated it. + It is now possible to do precisely, using ``Literal[DiffConstants.INDEX]``. """ def _process_diff_args( @@ -213,10 +229,9 @@ def diff( A :class:`DiffIndex` representing the computed diff. :note: - On a bare repository, `other` needs to be provided as - :class:`~Diffable.Index`, or as an instance of - :class:`~git.objects.tree.Tree` or :class:`~git.objects.commit.Commit`, or a - git command error will occur. + On a bare repository, `other` needs to be provided as :const:`INDEX`, or as + an instance of :class:`~git.objects.tree.Tree` or + :class:`~git.objects.commit.Commit`, or a git command error will occur. """ args: List[Union[PathLike, Diffable]] = [] args.append("--abbrev=40") # We need full shas. From b8a25dfbe1c7a3811549ab2fd4e10482af70f752 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 15:10:58 -0500 Subject: [PATCH 0782/1392] Modify annotations to accommodate NULL_TREE This finishes the main changes being done at this time to the annotations of diff-related methods that were started in 0e1df29. As expected and noted in previous commits, having Diffable.diff permit NULL_TREE entails a violation of the Liskov substitution principle in the overridden method IndexFile.diff unless a similar change were also made there, which could not be done correctly without modifying its behavior to actually accept NULL_TREE (it does not contain code to cover it, and raises ane exception if it is passed). I am unsure if that should ultimately be done, but even if so, it seems to me to be beyond the scope of the typing changes being done here. This therefore applies a suppression there. The suppression is specific to that one parameter. The long-standing comment atop the IndexFile.diff method, which reads as vague today, is replaced with a specific FIXME comment describing the situation where the method refers to the base-class docstring for documentation of its parameters yet doesn't accept NULL_TREE (and notes the mypy error). In effect this is really restoring and fixing the suppression that was present before 0e1df29 rather than adding a "new" one, but at that time the base-class parameter type was much broader since it was a union with object as one of its alternatives, so the situation was much less clear. --- git/diff.py | 2 +- git/index/base.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/git/diff.py b/git/diff.py index e185b4cb3..06935f87e 100644 --- a/git/diff.py +++ b/git/diff.py @@ -189,7 +189,7 @@ def _process_diff_args( def diff( self, - other: Union[Literal[DiffConstants.INDEX], "Tree", "Commit", str, None] = INDEX, + other: Union[DiffConstants, "Tree", "Commit", str, None] = INDEX, paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, **kwargs: Any, diff --git a/git/index/base.py b/git/index/base.py index 48e843822..dd58200ed 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1475,10 +1475,13 @@ def reset( return self - # @ default_index, breaks typing for some reason, copied into function + # FIXME: This is documented to accept the same parameters as Diffable.diff, but this + # does not handle NULL_TREE for `other`. (The suppressed mypy error is about this.) def diff( self, - other: Union[Literal[git_diff.DiffConstants.INDEX], "Tree", "Commit", str, None] = git_diff.INDEX, + other: Union[ # type: ignore[override] + Literal[git_diff.DiffConstants.INDEX], "Tree", "Commit", str, None + ] = git_diff.INDEX, paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, **kwargs: Any, From e49327dbe5ecd792d3c354155b6f186b31409c33 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 23 Feb 2024 19:51:39 -0500 Subject: [PATCH 0783/1392] Add refresh to top-level __all__ This makes the git.refresh function unambiguously public. git.refresh was already public in the sense that it was explicitly documented as appropriate to call from code outside GitPython. However, it had not been included in git.__all__. Because __all__ existed but omitted "refresh", git.refresh had appeared non-public to automated tools. This also does some cleanup: - It removes a comment that showed how git.__all__ had been defined dynamically before #1659, since with the addition of "refresh", git.__all__ no longer contains exactly the same elements as that technique produced (as it examined the module's contents prior to running the def statement that bound the name "refresh"). - With that comment removed, it is no longer necessary to define __all__ below the imports to show what the dynamic techinque had operated on. So this moves it up above them in accordance with PEP-8. --- git/__init__.py | 63 ++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index ed8a88d4b..ac211f9b9 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -5,38 +5,6 @@ # @PydevCodeAnalysisIgnore -__version__ = "git" - -from typing import List, Optional, Sequence, Tuple, Union, TYPE_CHECKING - -from gitdb.util import to_hex_sha -from git.exc import * # noqa: F403 # @NoMove @IgnorePep8 -from git.types import PathLike - -try: - from git.compat import safe_decode # @NoMove @IgnorePep8 - from git.config import GitConfigParser # @NoMove @IgnorePep8 - from git.objects import * # noqa: F403 # @NoMove @IgnorePep8 - from git.refs import * # noqa: F403 # @NoMove @IgnorePep8 - from git.diff import * # noqa: F403 # @NoMove @IgnorePep8 - from git.db import * # noqa: F403 # @NoMove @IgnorePep8 - from git.cmd import Git # @NoMove @IgnorePep8 - from git.repo import Repo # @NoMove @IgnorePep8 - from git.remote import * # noqa: F403 # @NoMove @IgnorePep8 - from git.index import * # noqa: F403 # @NoMove @IgnorePep8 - from git.util import ( # @NoMove @IgnorePep8 - LockFile, - BlockingLockFile, - Stats, - Actor, - remove_password_if_present, - rmtree, - ) -except GitError as _exc: # noqa: F405 - raise ImportError("%s: %s" % (_exc.__class__.__name__, _exc)) from _exc - -# __all__ must be statically defined by py.typed support -# __all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] __all__ = [ # noqa: F405 "Actor", "AmbiguousObjectName", @@ -109,12 +77,43 @@ "UnsupportedOperation", "UpdateProgress", "WorkTreeRepositoryUnsupported", + "refresh", "remove_password_if_present", "rmtree", "safe_decode", "to_hex_sha", ] +__version__ = "git" + +from typing import List, Optional, Sequence, Tuple, Union, TYPE_CHECKING + +from gitdb.util import to_hex_sha +from git.exc import * # noqa: F403 # @NoMove @IgnorePep8 +from git.types import PathLike + +try: + from git.compat import safe_decode # @NoMove @IgnorePep8 + from git.config import GitConfigParser # @NoMove @IgnorePep8 + from git.objects import * # noqa: F403 # @NoMove @IgnorePep8 + from git.refs import * # noqa: F403 # @NoMove @IgnorePep8 + from git.diff import * # noqa: F403 # @NoMove @IgnorePep8 + from git.db import * # noqa: F403 # @NoMove @IgnorePep8 + from git.cmd import Git # @NoMove @IgnorePep8 + from git.repo import Repo # @NoMove @IgnorePep8 + from git.remote import * # noqa: F403 # @NoMove @IgnorePep8 + from git.index import * # noqa: F403 # @NoMove @IgnorePep8 + from git.util import ( # @NoMove @IgnorePep8 + LockFile, + BlockingLockFile, + Stats, + Actor, + remove_password_if_present, + rmtree, + ) +except GitError as _exc: # noqa: F405 + raise ImportError("%s: %s" % (_exc.__class__.__name__, _exc)) from _exc + # { Initialize git executable path GIT_OK = None From c8ad3a36d39adddb36f62722678cdada49f94708 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 23 Feb 2024 22:07:24 -0500 Subject: [PATCH 0784/1392] Deprecate public access to typing imports in git This adds comments to entries in git.__all__ for each of the entries that come from the standard library typing module, noting them as deprecated. These imports were included in __all__ inadvertently due to the way __all__ was dynamically constructed, and placed in __all__ explicitly when __all__ became static in #1659. They are there for backward compatibility, in case some code relies on them being there. But a module is unlikely to rely intentionally on the git module providing them, since they are not conceptually related to GitPython. `from git import *` should not typically be used, since wildcard imports are not generally recommended, as discussed in PEP-8. But if someone does choose to use it, they would probably benefit less from DeprecationWarning being issued for each of those names than they would usually benefit from DeprecationWarning. This could lead to developers deciding not to enable DeprecationWarning when it may otherwise be useful. For this reason, no attempt is currently made to issue DeprecationWarning when those names are accessed as attributes of the git module. --- git/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index ac211f9b9..d6a247de0 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -38,13 +38,13 @@ "IndexObject", "InvalidDBRoot", "InvalidGitRepositoryError", - "List", + "List", # Deprecated - import this from `typing` instead. "LockFile", "NULL_TREE", "NoSuchPathError", "ODBError", "Object", - "Optional", + "Optional", # Deprecated - import this from `typing` instead. "ParseError", "PathLike", "PushInfo", @@ -58,19 +58,19 @@ "RepositoryDirtyError", "RootModule", "RootUpdateProgress", - "Sequence", + "Sequence", # Deprecated - import this from `typing` instead. "StageType", "Stats", "Submodule", "SymbolicReference", - "TYPE_CHECKING", + "TYPE_CHECKING", # Deprecated - import this from `typing` instead. "Tag", "TagObject", "TagReference", "Tree", "TreeModifier", - "Tuple", - "Union", + "Tuple", # Deprecated - import this from `typing` instead. + "Union", # Deprecated - import this from `typing` instead. "UnmergedEntriesError", "UnsafeOptionError", "UnsafeProtocolError", From 3c8cbe90abb0f8b38bf53b111fac3e1d1e0a8de3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 4 Mar 2024 22:52:16 -0500 Subject: [PATCH 0785/1392] Mention collections.abc for Sequence In the deprecation comment for importing Sequence from the top level git module. Although the git module imports it from typing, collections.abc.Sequence supports __class_getitem__ since Python 3.9 (and typing.Sequence is actually itself deprecated since then). --- git/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/__init__.py b/git/__init__.py index d6a247de0..24b8054f0 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -58,7 +58,7 @@ "RepositoryDirtyError", "RootModule", "RootUpdateProgress", - "Sequence", # Deprecated - import this from `typing` instead. + "Sequence", # Deprecated - import from `typing`, or `collections.abc` in 3.9+. "StageType", "Stats", "Submodule", From 87b314ee1374e0c295ba65248b2c69c6c15378c7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 16:24:05 -0500 Subject: [PATCH 0786/1392] Add INDEX and DiffConstants to git.__all__ This adds the names of git.diff.INDEX and git.diff.DiffConstants to the top-level __all__. In particular, the recently introduced INDEX constant has the same standing as NULL_TREE, which was (rightly) already listed, and this change makes true the claims in some docstrings in the git.diff module that say that INDEX is accessible (among other ways) as git.INDEX. --- git/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git/__init__.py b/git/__init__.py index 24b8054f0..026dc1461 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -20,6 +20,7 @@ "CommandError", "Commit", "Diff", + "DiffConstants", "DiffIndex", "Diffable", "FetchInfo", @@ -33,6 +34,7 @@ "HEAD", "Head", "HookExecutionError", + "INDEX", "IndexEntry", "IndexFile", "IndexObject", From 9ed904cf6d3c36192161cded9615fefc2bf18bf7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 16:47:11 -0500 Subject: [PATCH 0787/1392] Adjust mypy options to work well with mypy 1.9.0 mypy 1.9.0 was very recently released. This makes two changes, in light of that, while avoiding breaking earlier versions: - Change the Python version mypy assumes from 3.7 to 3.8 as configiured in pyproject.toml, because starting in 1.9.0 mypy no longer supports type checking for Python 3.7. It makes sense to keep having a low version checked on local runs of mypy (when not further overridden with `--python-version`). Since the CI mypy steps currently are written not to fail their jobs if they fail (and they do fail, as there remain some mypy errors that are neither fixed nor suppressed), mypy status from CI is not obvious, and it may be that the main way mypy is actually used when developing GitPython is locally. But keeping it down to 3.7 prints a warning and falls back to the version of Python that is used to run mypy, starting in mypy 1.9.0. - Have each CI job that runs mypy do so for the Python version that job exists to test. (This is already the case for the *platform* that job exists to test, because the platform mypy should check for is not overridden in pyproject.toml.) This is done by passing the test matrix Python version with `--python-version`, which takes priority over the pyproject.toml option (which itself takes priority over the default version selection, which in this case would be the same as what we are turning it back into on CI). This is worthwhile even separately from the changes in mypy 1.9.0. But one of its effects is to make it so the mypy step in the Python 3.7 test jobs still checks Python 3.7 (so the results for 3.7, which GitPython has not yet dropped support for) can still be examined. This works becuase the latest version of mypy that can *run* on 3.7, which pip selects automatically, *does* support type-checking for 3.7. --- .github/workflows/pythonpackage.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 6b89530c3..e43f763e2 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -88,7 +88,7 @@ jobs: - name: Check types with mypy run: | - mypy -p git + mypy --python-version=${{ matrix.python-version }} -p git # With new versions of mypy new issues might arise. This is a problem if there is nobody able to fix them, # so we have to ignore errors until that changes. continue-on-error: true diff --git a/pyproject.toml b/pyproject.toml index 7109389d7..03c68a5c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ testpaths = "test" # Space separated list of paths from root e.g test tests doc # filterwarnings ignore::WarningType # ignores those warnings [tool.mypy] -python_version = "3.7" +python_version = "3.8" disallow_untyped_defs = true no_implicit_optional = true warn_redundant_casts = true From aeacb001e9d316ffcc054761c033eb3c107f03e1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 17:09:54 -0500 Subject: [PATCH 0788/1392] Colorize mypy output on CI for easier reading See: - https://github.com/python/mypy/issues/13815 - https://github.com/python/mypy/issues/13817 This seems to be working in in all jobs *except* for the Python 3.7 and 3.8 jobs on macOS. --- .github/workflows/pythonpackage.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index e43f763e2..604110f70 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -89,8 +89,11 @@ jobs: - name: Check types with mypy run: | mypy --python-version=${{ matrix.python-version }} -p git - # With new versions of mypy new issues might arise. This is a problem if there is nobody able to fix them, - # so we have to ignore errors until that changes. + env: + MYPY_FORCE_COLOR: "1" + TERM: "xterm-256color" # For color: https://github.com/python/mypy/issues/13817 + # With new versions of mypy new issues might arise. This is a problem if there is + # nobody able to fix them, so we have to ignore errors until that changes. continue-on-error: true - name: Test with pytest From 84fc8062dccbf9c5e415f4ce0100e0a18680bca8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 19:21:11 -0500 Subject: [PATCH 0789/1392] Remove some unneeded mypy suppressions These include suppressions that are no longer required because they worked around mypy bugs that have been fixed, or because they were not actually effective. It may also include some that are no longer needed becuase of improvements made to GitPython's type annotations (or other code) since they were introduced, I am not sure. This deliberately keeps warn_unused_ignores uncommented in pyproject.toml, at least for now. --- git/config.py | 4 ++-- git/objects/commit.py | 2 +- git/objects/tree.py | 4 ++-- git/objects/util.py | 2 +- git/refs/reference.py | 4 ++-- git/util.py | 4 ++-- pyproject.toml | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/git/config.py b/git/config.py index 2164f67dc..78a88ba5e 100644 --- a/git/config.py +++ b/git/config.py @@ -344,9 +344,9 @@ def __init__( configuration files. """ cp.RawConfigParser.__init__(self, dict_type=_OMD) - self._dict: Callable[..., _OMD] # type: ignore # mypy/typeshed bug? + self._dict: Callable[..., _OMD] self._defaults: _OMD - self._sections: _OMD # type: ignore # mypy/typeshed bug? + self._sections: _OMD # Used in Python 3. Needs to stay in sync with sections for underlying # implementation to work. diff --git a/git/objects/commit.py b/git/objects/commit.py index 3246d9a36..28df86c9d 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -292,7 +292,7 @@ def name_rev(self) -> str: def iter_items( cls, repo: "Repo", - rev: Union[str, "Commit", "SymbolicReference"], # type: ignore + rev: Union[str, "Commit", "SymbolicReference"], paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any, ) -> Iterator["Commit"]: diff --git a/git/objects/tree.py b/git/objects/tree.py index 995c9a663..385389f03 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -300,7 +300,7 @@ def cache(self) -> TreeModifier: return TreeModifier(self._cache) def traverse( - self, # type: ignore[override] + self, predicate: Callable[[Union[IndexObjUnion, TraversedTreeTup], int], bool] = lambda i, d: True, prune: Callable[[Union[IndexObjUnion, TraversedTreeTup], int], bool] = lambda i, d: False, depth: int = -1, @@ -331,7 +331,7 @@ def traverse( super()._traverse( predicate, prune, - depth, # type: ignore + depth, branch_first, visit_once, ignore_self, diff --git a/git/objects/util.py b/git/objects/util.py index 6f4e7d087..194403ba5 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -619,7 +619,7 @@ class TraversableIterableObj(IterableObj, Traversable): def list_traverse(self: T_TIobj, *args: Any, **kwargs: Any) -> IterableList[T_TIobj]: return super()._list_traverse(*args, **kwargs) - @overload # type: ignore + @overload def traverse(self: T_TIobj) -> Iterator[T_TIobj]: ... diff --git a/git/refs/reference.py b/git/refs/reference.py index a7b545fed..2da511ce3 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -150,7 +150,7 @@ def iter_items( # { Remote Interface - @property # type: ignore # mypy cannot deal with properties with an extra decorator (2021-04-21). + @property @require_remote_ref_path def remote_name(self) -> str: """ @@ -162,7 +162,7 @@ def remote_name(self) -> str: # /refs/remotes// return tokens[2] - @property # type: ignore # mypy cannot deal with properties with an extra decorator (2021-04-21). + @property @require_remote_ref_path def remote_head(self) -> str: """ diff --git a/git/util.py b/git/util.py index 880d6625a..d9e223c76 100644 --- a/git/util.py +++ b/git/util.py @@ -510,8 +510,8 @@ def expand_path(p: Union[None, PathLike], expand_vars: bool = True) -> Optional[ try: p = osp.expanduser(p) # type: ignore if expand_vars: - p = osp.expandvars(p) # type: ignore - return osp.normpath(osp.abspath(p)) # type: ignore + p = osp.expandvars(p) + return osp.normpath(osp.abspath(p)) except Exception: return None diff --git a/pyproject.toml b/pyproject.toml index 03c68a5c6..2aeae59f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ python_version = "3.8" disallow_untyped_defs = true no_implicit_optional = true warn_redundant_casts = true -# warn_unused_ignores = true +warn_unused_ignores = true warn_unreachable = true show_error_codes = true implicit_reexport = true From 96ecc2edeb14b950737d7f6937055c2b19ea5d5a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 19:29:16 -0500 Subject: [PATCH 0790/1392] Drop deprecated mypy option show_error_codes is deprecated, and represents the default now. --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2aeae59f9..ef1189b0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,6 @@ no_implicit_optional = true warn_redundant_casts = true warn_unused_ignores = true warn_unreachable = true -show_error_codes = true implicit_reexport = true # strict = true From 97d9b65937ffe9b7ce374dccdbfd4ef24b327b4a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 20:00:46 -0500 Subject: [PATCH 0791/1392] Apply intended suppression in Tree.traverse The superclass _traverse method call in Tree.traverse appears to have once had a working suppression for the incompatbile types of two arguments, `predicate` and `prune`, as well as for an argument that required (and requires) no suppression, `depth`. These three arguments were written on the same line, which bad a `type: ignore` comment on it. But when black formatting was applied in 21ec529 (#1442), that comment moved so that it was on a line with just the `depth` call that didn't need it, rather than the others that did. Since then, mypy has reported errors, which further seem intended to suppress based on the surrounding context and the use of `cast` to deal with the static type incompatibilities going the other way. This misplaced suppression was one of the ones I very recently removed in 84fc806. But really there should be a suppression for those arguments (at least for now, while the code remains written that way, given that a suppression is intended). This suppresses the error effectively by inserting two suppression comments, one for each of the two arguments. This is more specific than a single suppression applying to the whole call, and keeping the arguments on separate lines both makes black happy and makes clear that it is not by coincidence that the error is suppressed for both of them. The new suppressions are also written for the specific mypy error at issue, rather than fully general as before. This change decreases the number of mypy errors by two. --- git/objects/tree.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/objects/tree.py b/git/objects/tree.py index 385389f03..30b6a9e4e 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -329,8 +329,8 @@ def traverse( return cast( Union[Iterator[IndexObjUnion], Iterator[TraversedTreeTup]], super()._traverse( - predicate, - prune, + predicate, # type: ignore[arg-type] + prune, # type: ignore[arg-type] depth, branch_first, visit_once, From ad00c77b39707ca9a347780a18701e342128bcf2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 20:41:59 -0500 Subject: [PATCH 0792/1392] Spell self.Index as self.INDEX in IndexFile.diff Because, as noted in 65863a2, it is now (slightly) preferable to write INDEX. Note that this is solely stylstic: they are equivalent and must remain so, so that existing code that uses GitPython and accesses it as Index (including in overridden diff methods when subclassing Diffable) continues to work. This changes it in the exception message as well. --- git/index/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index dd58200ed..a95c3a622 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1500,7 +1500,7 @@ def diff( if self._file_path != self._index_path(): raise AssertionError("Cannot call %r on indices that do not represent the default git index" % self.diff()) # Index against index is always empty. - if other is self.Index: + if other is self.INDEX: return git_diff.DiffIndex() # Index against anything but None is a reverse diff with the respective item. @@ -1514,12 +1514,12 @@ def diff( # Invert the existing R flag. cur_val = kwargs.get("R", False) kwargs["R"] = not cur_val - return other.diff(self.Index, paths, create_patch, **kwargs) + return other.diff(self.INDEX, paths, create_patch, **kwargs) # END diff against other item handling # If other is not None here, something is wrong. if other is not None: - raise ValueError("other must be None, Diffable.Index, a Tree or Commit, was %r" % other) + raise ValueError("other must be None, Diffable.INDEX, a Tree or Commit, was %r" % other) # Diff against working copy - can be handled by superclass natively. return super().diff(other, paths, create_patch, **kwargs) From 2decbe45019fc8dcc6904fcdc6f1088f52b9f00c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 20:57:59 -0500 Subject: [PATCH 0793/1392] Test that redefined Diffable.Index should be compatible This tests the most important characteristics of Diffable.Index that must be (and are) preserved in making it an alias of the enumeration constant INDEX instead of a nested class definition. Adding this additional test also allows commit.INDEX to be used in place of commit.Index in the existing test (since part of what this tests is that they are the same object). That change is also made, along with some very minor cleanup of immediately nearby test code. --- test/test_diff.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/test/test_diff.py b/test/test_diff.py index ed82b1bbd..6d2f82649 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -12,7 +12,7 @@ import ddt import pytest -from git import NULL_TREE, Diff, DiffIndex, GitCommandError, Repo, Submodule +from git import NULL_TREE, Diff, DiffIndex, Diffable, GitCommandError, Repo, Submodule from git.cmd import Git from test.lib import StringProcessAdapter, TestBase, fixture, with_rw_directory @@ -352,7 +352,7 @@ def test_diff_submodule(self): self.assertIsInstance(diff.b_blob.size, int) def test_diff_interface(self): - # Test a few variations of the main diff routine. + """Test a few variations of the main diff routine.""" assertion_map = {} for i, commit in enumerate(self.rorepo.iter_commits("0.1.6", max_count=2)): diff_item = commit @@ -360,7 +360,7 @@ def test_diff_interface(self): diff_item = commit.tree # END use tree every second item - for other in (None, NULL_TREE, commit.Index, commit.parents[0]): + for other in (None, NULL_TREE, commit.INDEX, commit.parents[0]): for paths in (None, "CHANGES", ("CHANGES", "lib")): for create_patch in range(2): diff_index = diff_item.diff(other=other, paths=paths, create_patch=create_patch) @@ -406,10 +406,22 @@ def test_diff_interface(self): diff_index = c.diff(cp, ["does/not/exist"]) self.assertEqual(len(diff_index), 0) + def test_diff_interface_stability(self): + """Test that the Diffable.Index redefinition should not break compatibility.""" + self.assertIs( + Diffable.Index, + Diffable.INDEX, + "The old and new class attribute names must be aliases.", + ) + self.assertIs( + type(Diffable.INDEX).__eq__, + object.__eq__, + "Equality comparison must be reference-based.", + ) + @with_rw_directory def test_rename_override(self, rw_dir): - """Test disabling of diff rename detection""" - + """Test disabling of diff rename detection.""" # Create and commit file_a.txt. repo = Repo.init(rw_dir) file_a = osp.join(rw_dir, "file_a.txt") From 88557bc066209db37101938236792bf0fdd6c535 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 Mar 2024 23:39:35 -0500 Subject: [PATCH 0794/1392] Have git module use sys.platform to check for Windows This changes the way code throughout the git module checks to see if it is running on a native Windows system. Some checks using os.name were already changed to use sys.platform in dc95a76 and 4191f7d. (See dc95a76 on why the specific question of whether the system is native Windows can be answered by either checking os.name or sys.platform, even though in general they differ in granularity and are not always suited to the same tasks.) The reasons for this change are: - Consistency: Due to dc95a76 and 4191f7d, some of these checks use sys.platform, so in the absence of a reason to do otherwise, it is best that they all do. Otherwise, it creates an appearance that the technical reasons behind the difference are stronger than they really are, or even that differnt things are being checked. - Better static typing: mypy treats sys.platform as a constant (and also allows checking on platform on another via --platform). Likewise, typeshed conditions platform-dependent declarations on sys.platform checks, rather than os.name or other checks. Really this is the original reason for those earlier, more selective changes, but here the goal is more general, since this is not needed to address any specific preexisting mypy errors. This is incomplete, for two reasons: - I'm deliberately not including changes in the tests in this commit. Arguably it should be kept as os.name in the tests, on the grounds that the test suite is not currently statically typed anyway, plus having them differ more compellingly shows that the behavior is the same whether an os.name or sys.platform check is used. However, it would be confusing to keep them different, and also somewhat unnatural; one approach would probably end up leaking through. Furthermore, because some tests have to check for cygwin specifically, which cannot be done with os.name, it may be clearer to use sys.platform for all platform checking in the test suite. But to verify and demonstrate that the change really is safe, I'm waiting to make the change in the tests until after making them in the code under test. - Some forms of checks against constants produce unreachable code errors from mypy (https://github.com/python/mypy/issues/10773). This can be worked around, but I have not done that in this commit. Furthermore, the new mypy errors this produces--one on Windows, and one on non-Windows systems--could be fixed in a way that makes type annotations richer, by allowing the return type to be a literal on one platform or the other. --- git/config.py | 2 +- git/index/base.py | 3 ++- git/index/fun.py | 3 ++- git/objects/submodule/base.py | 3 ++- git/repo/base.py | 7 ++++--- git/util.py | 18 ++++++++++-------- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/git/config.py b/git/config.py index 78a88ba5e..4441c2187 100644 --- a/git/config.py +++ b/git/config.py @@ -246,7 +246,7 @@ def items_all(self) -> List[Tuple[str, List[_T]]]: def get_config_path(config_level: Lit_config_levels) -> str: # We do not support an absolute path of the gitconfig on Windows. # Use the global config instead. - if os.name == "nt" and config_level == "system": + if sys.platform == "win32" and config_level == "system": config_level = "global" if config_level == "system": diff --git a/git/index/base.py b/git/index/base.py index a95c3a622..704235241 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -13,6 +13,7 @@ import os from stat import S_ISLNK import subprocess +import sys import tempfile from git.compat import ( @@ -107,7 +108,7 @@ def _named_temporary_file_for_subprocess(directory: PathLike) -> Generator[str, A context manager object that creates the file and provides its name on entry, and deletes it on exit. """ - if os.name == "nt": + if sys.platform == "win32": fd, name = tempfile.mkstemp(dir=directory) os.close(fd) try: diff --git a/git/index/fun.py b/git/index/fun.py index 58335739e..60483c1f4 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -18,6 +18,7 @@ S_IXUSR, ) import subprocess +import sys from git.cmd import handle_process_output, safer_popen from git.compat import defenc, force_bytes, force_text, safe_decode @@ -99,7 +100,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: env["GIT_EDITOR"] = ":" cmd = [hp] try: - if os.name == "nt" and not _has_file_extension(hp): + if sys.platform == "win32" and not _has_file_extension(hp): # Windows only uses extensions to determine how to open files # (doesn't understand shebangs). Try using bash to run the hook. relative_hp = Path(hp).relative_to(index.repo.working_dir).as_posix() diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index cdd7c8e1b..a7d638c59 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -7,6 +7,7 @@ import os import os.path as osp import stat +import sys import uuid import git @@ -401,7 +402,7 @@ def _write_git_file_and_module_config(cls, working_tree_dir: PathLike, module_ab """ git_file = osp.join(working_tree_dir, ".git") rela_path = osp.relpath(module_abspath, start=working_tree_dir) - if os.name == "nt" and osp.isfile(git_file): + if sys.platform == "win32" and osp.isfile(git_file): os.remove(git_file) with open(git_file, "wb") as fp: fp.write(("gitdir: %s" % rela_path).encode(defenc)) diff --git a/git/repo/base.py b/git/repo/base.py index 39a80b449..9b42ec85a 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -12,6 +12,7 @@ from pathlib import Path import re import shlex +import sys import warnings import gitdb @@ -336,10 +337,10 @@ def close(self) -> None: # they are collected by the garbage collector, thus preventing deletion. # TODO: Find these references and ensure they are closed and deleted # synchronously rather than forcing a gc collection. - if os.name == "nt": + if sys.platform == "win32": gc.collect() gitdb.util.mman.collect() - if os.name == "nt": + if sys.platform == "win32": gc.collect() def __eq__(self, rhs: object) -> bool: @@ -618,7 +619,7 @@ def _get_config_path(self, config_level: Lit_config_levels, git_dir: Optional[Pa git_dir = self.git_dir # We do not support an absolute path of the gitconfig on Windows. # Use the global config instead. - if os.name == "nt" and config_level == "system": + if sys.platform == "win32" and config_level == "system": config_level = "global" if config_level == "system": diff --git a/git/util.py b/git/util.py index d9e223c76..a0c1d61e5 100644 --- a/git/util.py +++ b/git/util.py @@ -117,7 +117,7 @@ def _read_win_env_flag(name: str, default: bool) -> bool: :note: This only accesses the environment on Windows. """ - if os.name != "nt": + if sys.platform != "win32": return False try: @@ -223,7 +223,7 @@ def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: raise SkipTest(f"FIXME: fails with: PermissionError\n {ex}") from ex raise - if os.name != "nt": + if sys.platform != "win32": shutil.rmtree(path) elif sys.version_info >= (3, 12): shutil.rmtree(path, onexc=handler) @@ -235,7 +235,7 @@ def rmfile(path: PathLike) -> None: """Ensure file deleted also on *Windows* where read-only files need special treatment.""" if osp.isfile(path): - if os.name == "nt": + if sys.platform == "win32": os.chmod(path, 0o777) os.remove(path) @@ -276,7 +276,7 @@ def join_path(a: PathLike, *p: PathLike) -> PathLike: return path -if os.name == "nt": +if sys.platform == "win32": def to_native_path_windows(path: PathLike) -> PathLike: path = str(path) @@ -328,7 +328,7 @@ def _get_exe_extensions() -> Sequence[str]: PATHEXT = os.environ.get("PATHEXT", None) if PATHEXT: return tuple(p.upper() for p in PATHEXT.split(os.pathsep)) - elif os.name == "nt": + elif sys.platform == "win32": return (".BAT", "COM", ".EXE") else: return () @@ -354,7 +354,9 @@ def is_exec(fpath: str) -> bool: return ( osp.isfile(fpath) and os.access(fpath, os.X_OK) - and (os.name != "nt" or not winprog_exts or any(fpath.upper().endswith(ext) for ext in winprog_exts)) + and ( + sys.platform != "win32" or not winprog_exts or any(fpath.upper().endswith(ext) for ext in winprog_exts) + ) ) progs = [] @@ -451,8 +453,8 @@ def is_cygwin_git(git_executable: PathLike) -> bool: def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: - if os.name == "nt": - # This is Windows-native Python, since Cygwin has os.name == "posix". + if sys.platform == "win32": + # This is Windows-native Python, since Cygwin has sys.platform == "cygwin". return False if git_executable is None: From 7204c131fdcb4132e34a4f6d4c38af71083afdb1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 00:13:55 -0500 Subject: [PATCH 0795/1392] Fix new mypy error in _read_win_env_flag As noted in 88557bc before this, that change added a couple of new mypy errors about unreachable code, where it is intentionally unreachable because it is for one platform and not another. This fixes one of them, though a fix that incorporates more of what can now be statically known about the return value based on the platform may be preferable. --- git/util.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/git/util.py b/git/util.py index a0c1d61e5..92f430777 100644 --- a/git/util.py +++ b/git/util.py @@ -107,19 +107,12 @@ _logger = logging.getLogger(__name__) -def _read_win_env_flag(name: str, default: bool) -> bool: - """Read a boolean flag from an environment variable on Windows. +def _read_env_flag(name: str, default: bool) -> bool: + """Read a boolean flag from an environment variable. :return: - On Windows, the flag, or the `default` value if absent or ambiguous. - On all other operating systems, ``False``. - - :note: - This only accesses the environment on Windows. + The flag, or the `default` value if absent or ambiguous. """ - if sys.platform != "win32": - return False - try: value = os.environ[name] except KeyError: @@ -140,6 +133,19 @@ def _read_win_env_flag(name: str, default: bool) -> bool: return default +def _read_win_env_flag(name: str, default: bool) -> bool: + """Read a boolean flag from an environment variable on Windows. + + :return: + On Windows, the flag, or the `default` value if absent or ambiguous. + On all other operating systems, ``False``. + + :note: + This only accesses the environment on Windows. + """ + return sys.platform == "win32" and _read_env_flag(name, default) + + #: We need an easy way to see if Appveyor TCs start failing, #: so the errors marked with this var are considered "acknowledged" ones, awaiting remedy, #: till then, we wish to hide them. From 42e10c07436cd6186ffff1861c466838366fe9f0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 01:45:24 -0500 Subject: [PATCH 0796/1392] Fix new mypy error in is_cygwin_git This fixes the second new mypy error that arose due to the change in 88557bc. (7204c13 before this fixed the first of them.) --- git/util.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/git/util.py b/git/util.py index 92f430777..7ebce0c2c 100644 --- a/git/util.py +++ b/git/util.py @@ -448,25 +448,7 @@ def decygpath(path: PathLike) -> str: _is_cygwin_cache: Dict[str, Optional[bool]] = {} -@overload -def is_cygwin_git(git_executable: None) -> Literal[False]: - ... - - -@overload -def is_cygwin_git(git_executable: PathLike) -> bool: - ... - - -def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: - if sys.platform == "win32": - # This is Windows-native Python, since Cygwin has sys.platform == "cygwin". - return False - - if git_executable is None: - return False - - git_executable = str(git_executable) +def _is_cygwin_git(git_executable: str) -> bool: is_cygwin = _is_cygwin_cache.get(git_executable) # type: Optional[bool] if is_cygwin is None: is_cygwin = False @@ -489,6 +471,25 @@ def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: return is_cygwin +@overload +def is_cygwin_git(git_executable: None) -> Literal[False]: + ... + + +@overload +def is_cygwin_git(git_executable: PathLike) -> bool: + ... + + +def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: + if sys.platform == "win32": # TODO: See if we can use `sys.platform != "cygwin"`. + return False + elif git_executable is None: + return False + else: + return _is_cygwin_git(str(git_executable)) + + def get_user_id() -> str: """:return: String identifying the currently active system user as ``name@node``""" return "%s@%s" % (getpass.getuser(), platform.node()) From 465ab56b25261dc5830b3c4b91b39c7346c527df Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 01:52:35 -0500 Subject: [PATCH 0797/1392] Have test suite use sys.platform to check for Windows See 88557bc on this change in general as well as the rationale for making it in the tests. (It is alreay done in the code under test.) --- test/lib/helper.py | 9 +++++---- test/test_base.py | 2 +- test/test_config.py | 3 ++- test/test_diff.py | 4 ++-- test/test_git.py | 10 +++++----- test/test_index.py | 11 ++++++----- test/test_remote.py | 10 +++++----- test/test_repo.py | 6 +++--- test/test_submodule.py | 4 ++-- test/test_util.py | 8 ++++---- 10 files changed, 35 insertions(+), 32 deletions(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index 26469ed5d..0a6df242c 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -10,6 +10,7 @@ import logging import os import os.path as osp +import sys import tempfile import textwrap import time @@ -176,7 +177,7 @@ def git_daemon_launched(base_path, ip, port): gd = None try: - if os.name == "nt": + if sys.platform == "win32": # On MINGW-git, daemon exists in Git\mingw64\libexec\git-core\, # but if invoked as 'git daemon', it detaches from parent `git` cmd, # and then CANNOT DIE! @@ -200,7 +201,7 @@ def git_daemon_launched(base_path, ip, port): as_process=True, ) # Yes, I know... fortunately, this is always going to work if sleep time is just large enough. - time.sleep(1.0 if os.name == "nt" else 0.5) + time.sleep(1.0 if sys.platform == "win32" else 0.5) except Exception as ex: msg = textwrap.dedent( """ @@ -404,7 +405,7 @@ class VirtualEnvironment: __slots__ = ("_env_dir",) def __init__(self, env_dir, *, with_pip): - if os.name == "nt": + if sys.platform == "win32": self._env_dir = osp.realpath(env_dir) venv.create(self.env_dir, symlinks=False, with_pip=with_pip) else: @@ -432,7 +433,7 @@ def sources(self): return os.path.join(self.env_dir, "src") def _executable(self, basename): - if os.name == "nt": + if sys.platform == "win32": path = osp.join(self.env_dir, "Scripts", basename + ".exe") else: path = osp.join(self.env_dir, "bin", basename) diff --git a/test/test_base.py b/test/test_base.py index ef7486e86..e477b4837 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -130,7 +130,7 @@ def test_add_unicode(self, rw_repo): with open(file_path, "wb") as fp: fp.write(b"something") - if os.name == "nt": + if sys.platform == "win32": # On Windows, there is no way this works, see images on: # https://github.com/gitpython-developers/GitPython/issues/147#issuecomment-68881897 # Therefore, it must be added using the Python implementation. diff --git a/test/test_config.py b/test/test_config.py index 4843d91eb..ac19a7fa8 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -7,6 +7,7 @@ import io import os import os.path as osp +import sys from unittest import mock import pytest @@ -238,7 +239,7 @@ def check_test_value(cr, value): check_test_value(cr, tv) @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason='Second config._has_includes() assertion fails (for "config is included if path is matching git_dir")', raises=AssertionError, ) diff --git a/test/test_diff.py b/test/test_diff.py index 6d2f82649..96fbc60e3 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -4,9 +4,9 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import gc -import os import os.path as osp import shutil +import sys import tempfile import ddt @@ -309,7 +309,7 @@ def test_diff_with_spaces(self): self.assertEqual(diff_index[0].b_path, "file with spaces", repr(diff_index[0].b_path)) @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason='"Access is denied" when tearDown calls shutil.rmtree', raises=PermissionError, ) diff --git a/test/test_git.py b/test/test_git.py index e1a8bda5e..dae0f6a39 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -74,7 +74,7 @@ def _fake_git(*version_info): fake_output = f"git version {fake_version} (fake)" with tempfile.TemporaryDirectory() as tdir: - if os.name == "nt": + if sys.platform == "win32": fake_git = Path(tdir, "fake-git.cmd") script = f"@echo {fake_output}\n" fake_git.write_text(script, encoding="utf-8") @@ -215,7 +215,7 @@ def test_it_executes_git_not_from_cwd(self, rw_dir, case): repo = Repo.init(rw_dir) - if os.name == "nt": + if sys.platform == "win32": # Copy an actual binary executable that is not git. (On Windows, running # "hostname" only displays the hostname, it never tries to change it.) other_exe_path = Path(os.environ["SystemRoot"], "system32", "hostname.exe") @@ -228,7 +228,7 @@ def test_it_executes_git_not_from_cwd(self, rw_dir, case): os.chmod(impostor_path, 0o755) if use_shell_impostor: - shell_name = "cmd.exe" if os.name == "nt" else "sh" + shell_name = "cmd.exe" if sys.platform == "win32" else "sh" shutil.copy(impostor_path, Path(rw_dir, shell_name)) with contextlib.ExitStack() as stack: @@ -245,7 +245,7 @@ def test_it_executes_git_not_from_cwd(self, rw_dir, case): self.assertRegex(output, r"^git version\b") @skipUnless( - os.name == "nt", + sys.platform == "win32", "The regression only affected Windows, and this test logic is OS-specific.", ) def test_it_avoids_upcasing_unrelated_environment_variable_names(self): @@ -667,7 +667,7 @@ def test_successful_default_refresh_invalidates_cached_version_info(self): stack.enter_context(mock.patch.dict(os.environ, {"PATH": new_path_var})) stack.enter_context(_patch_out_env("GIT_PYTHON_GIT_EXECUTABLE")) - if os.name == "nt": + if sys.platform == "win32": # On Windows, use a shell so "git" finds "git.cmd". (In the infrequent # case that this effect is desired in production code, it should not be # done with this technique. USE_SHELL=True is less secure and reliable, diff --git a/test/test_index.py b/test/test_index.py index fa64b82a2..622e7ca9a 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -14,6 +14,7 @@ import shutil from stat import S_ISLNK, ST_MODE import subprocess +import sys import tempfile import ddt @@ -124,7 +125,7 @@ def check(cls): in System32; Popen finds it even if a shell would run another one, as on CI. (Without WSL, System32 may still have bash.exe; users sometimes put it there.) """ - if os.name != "nt": + if sys.platform != "win32": return cls.Inapplicable() try: @@ -561,7 +562,7 @@ def _count_existing(self, repo, files): # END num existing helper @pytest.mark.xfail( - os.name == "nt" and Git().config("core.symlinks") == "true", + sys.platform == "win32" and Git().config("core.symlinks") == "true", reason="Assumes symlinks are not created on Windows and opens a symlink to a nonexistent target.", raises=FileNotFoundError, ) @@ -754,7 +755,7 @@ def mixed_iterator(): self.assertNotEqual(entries[0].hexsha, null_hex_sha) # Add symlink. - if os.name != "nt": + if sys.platform != "win32": for target in ("/etc/nonexisting", "/etc/passwd", "/etc"): basename = "my_real_symlink" @@ -812,7 +813,7 @@ def mixed_iterator(): index.checkout(fake_symlink_path) # On Windows, we currently assume we will never get symlinks. - if os.name == "nt": + if sys.platform == "win32": # Symlinks should contain the link as text (which is what a # symlink actually is). with open(fake_symlink_path, "rt") as fd: @@ -1043,7 +1044,7 @@ def test_run_commit_hook(self, rw_repo): def test_hook_uses_shell_not_from_cwd(self, rw_dir, case): (chdir_to_repo,) = case - shell_name = "bash.exe" if os.name == "nt" else "sh" + shell_name = "bash.exe" if sys.platform == "win32" else "sh" maybe_chdir = cwd(rw_dir) if chdir_to_repo else contextlib.nullcontext() repo = Repo.init(rw_dir) diff --git a/test/test_remote.py b/test/test_remote.py index c0bd11f80..5344a7324 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -4,10 +4,10 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import gc -import os import os.path as osp from pathlib import Path import random +import sys import tempfile from unittest import skipIf @@ -769,7 +769,7 @@ def test_create_remote_unsafe_url(self, rw_repo): assert not tmp_file.exists() @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason=R"Multiple '\' instead of '/' in remote.url make it differ from expected value", raises=AssertionError, ) @@ -832,7 +832,7 @@ def test_fetch_unsafe_options(self, rw_repo): assert not tmp_file.exists() @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason=( "File not created. A separate Windows command may be needed. This and the " "currently passing test test_fetch_unsafe_options must be adjusted in the " @@ -900,7 +900,7 @@ def test_pull_unsafe_options(self, rw_repo): assert not tmp_file.exists() @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason=( "File not created. A separate Windows command may be needed. This and the " "currently passing test test_pull_unsafe_options must be adjusted in the " @@ -974,7 +974,7 @@ def test_push_unsafe_options(self, rw_repo): assert not tmp_file.exists() @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason=( "File not created. A separate Windows command may be needed. This and the " "currently passing test test_push_unsafe_options must be adjusted in the " diff --git a/test/test_repo.py b/test/test_repo.py index 30a44b6c1..238f94712 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -309,7 +309,7 @@ def test_clone_unsafe_options(self, rw_repo): assert not tmp_file.exists() @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason=( "File not created. A separate Windows command may be needed. This and the " "currently passing test test_clone_unsafe_options must be adjusted in the " @@ -388,7 +388,7 @@ def test_clone_from_unsafe_options(self, rw_repo): assert not tmp_file.exists() @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason=( "File not created. A separate Windows command may be needed. This and the " "currently passing test test_clone_from_unsafe_options must be adjusted in the " @@ -1389,7 +1389,7 @@ def test_do_not_strip_newline_in_stdout(self, rw_dir): self.assertEqual(r.git.show("HEAD:hello.txt", strip_newline_in_stdout=False), "hello\n") @pytest.mark.xfail( - os.name == "nt", + sys.platform == "win32", reason=R"fatal: could not create leading directories of '--upload-pack=touch C:\Users\ek\AppData\Local\Temp\tmpnantqizc\pwn': Invalid argument", # noqa: E501 raises=GitCommandError, ) diff --git a/test/test_submodule.py b/test/test_submodule.py index 68164729b..ee7795dbb 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -987,7 +987,7 @@ def test_rename(self, rwdir): # This is needed to work around a PermissionError on Windows, resembling others, # except new in Python 3.12. (*Maybe* this could be due to changes in CPython's # garbage collector detailed in https://github.com/python/cpython/issues/97922.) - if os.name == "nt" and sys.version_info >= (3, 12): + if sys.platform == "win32" and sys.version_info >= (3, 12): gc.collect() new_path = "renamed/myname" @@ -1071,7 +1071,7 @@ def test_branch_renames(self, rw_dir): assert sm_mod.commit() == sm_pfb.commit, "Now head should have been reset" assert sm_mod.head.ref.name == sm_pfb.name - @skipUnless(os.name == "nt", "Specifically for Windows.") + @skipUnless(sys.platform == "win32", "Specifically for Windows.") def test_to_relative_path_with_super_at_root_drive(self): class Repo: working_tree_dir = "D:\\" diff --git a/test/test_util.py b/test/test_util.py index 824b3ab3d..369896581 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -149,7 +149,7 @@ def _patch_for_wrapping_test(self, mocker, hide_windows_known_errors): mocker.patch.object(pathlib.Path, "chmod") @pytest.mark.skipif( - os.name != "nt", + sys.platform != "win32", reason="PermissionError is only ever wrapped on Windows", ) def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): @@ -168,7 +168,7 @@ def test_wraps_perm_error_if_enabled(self, mocker, permission_error_tmpdir): "hide_windows_known_errors", [ pytest.param(False), - pytest.param(True, marks=pytest.mark.skipif(os.name == "nt", reason="We would wrap on Windows")), + pytest.param(True, marks=pytest.mark.skipif(sys.platform == "win32", reason="We would wrap on Windows")), ], ) def test_does_not_wrap_perm_error_unless_enabled(self, mocker, permission_error_tmpdir, hide_windows_known_errors): @@ -214,7 +214,7 @@ def _run_parse(name, value): return ast.literal_eval(output) @pytest.mark.skipif( - os.name != "nt", + sys.platform != "win32", reason="These environment variables are only used on Windows.", ) @pytest.mark.parametrize( @@ -410,7 +410,7 @@ def test_blocking_lock_file(self): elapsed = time.time() - start extra_time = 0.02 - if os.name == "nt" or sys.platform == "cygwin": + if sys.platform in {"win32", "cygwin"}: extra_time *= 6 # Without this, we get indeterministic failures on Windows. elif sys.platform == "darwin": extra_time *= 18 # The situation on macOS is similar, but with more delay. From ad8190bbb41db9056aefa1a29176b196c4edb466 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 03:14:35 -0400 Subject: [PATCH 0798/1392] Wrap docstrings and comments in _safer_popen_windows It gained an indentation level in dc95a76, so its docstrings and comments were no longer wrapped to 88 columns as most docstrings and comments have been (absent a reason not to) since #1850. This wraps it, but some parts may benefit from some other adjustments. --- git/cmd.py | 61 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 9fd2c532d..d17c8836b 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -232,47 +232,54 @@ def _safer_popen_windows( env: Optional[Mapping[str, str]] = None, **kwargs: Any, ) -> Popen: - """Call :class:`subprocess.Popen` on Windows but don't include a CWD in the search. - - This avoids an untrusted search path condition where a file like ``git.exe`` in a - malicious repository would be run when GitPython operates on the repository. The - process using GitPython may have an untrusted repository's working tree as its - current working directory. Some operations may temporarily change to that directory - before running a subprocess. In addition, while by default GitPython does not run - external commands with a shell, it can be made to do so, in which case the CWD of - the subprocess, which GitPython usually sets to a repository working tree, can - itself be searched automatically by the shell. This wrapper covers all those cases. + """Call :class:`subprocess.Popen` on Windows but don't include a CWD in the + search. + + This avoids an untrusted search path condition where a file like ``git.exe`` in + a malicious repository would be run when GitPython operates on the repository. + The process using GitPython may have an untrusted repository's working tree as + its current working directory. Some operations may temporarily change to that + directory before running a subprocess. In addition, while by default GitPython + does not run external commands with a shell, it can be made to do so, in which + case the CWD of the subprocess, which GitPython usually sets to a repository + working tree, can itself be searched automatically by the shell. This wrapper + covers all those cases. :note: - This currently works by setting the :envvar:`NoDefaultCurrentDirectoryInExePath` - environment variable during subprocess creation. It also takes care of passing - Windows-specific process creation flags, but that is unrelated to path search. + This currently works by setting the + :envvar:`NoDefaultCurrentDirectoryInExePath` environment variable during + subprocess creation. It also takes care of passing Windows-specific process + creation flags, but that is unrelated to path search. :note: The current implementation contains a race condition on :attr:`os.environ`. - GitPython isn't thread-safe, but a program using it on one thread should ideally - be able to mutate :attr:`os.environ` on another, without unpredictable results. - See comments in https://github.com/gitpython-developers/GitPython/pull/1650. + GitPython isn't thread-safe, but a program using it on one thread should + ideally be able to mutate :attr:`os.environ` on another, without + unpredictable results. See comments in: + https://github.com/gitpython-developers/GitPython/pull/1650 """ - # CREATE_NEW_PROCESS_GROUP is needed for some ways of killing it afterwards. See: + # CREATE_NEW_PROCESS_GROUP is needed for some ways of killing it afterwards. # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal # https://docs.python.org/3/library/subprocess.html#subprocess.CREATE_NEW_PROCESS_GROUP creationflags = subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP - # When using a shell, the shell is the direct subprocess, so the variable must be - # set in its environment, to affect its search behavior. (The "1" can be any value.) + # When using a shell, the shell is the direct subprocess, so the variable must + # be set in its environment, to affect its search behavior. (The "1" can be any + # value.) if shell: - # The original may be immutable or reused by the caller. Make changes in a copy. + # The original may be immutable or reused by the caller. Make changes in a + # copy. env = {} if env is None else dict(env) env["NoDefaultCurrentDirectoryInExePath"] = "1" - # When not using a shell, the current process does the search in a CreateProcessW - # API call, so the variable must be set in our environment. With a shell, this is - # unnecessary, in versions where https://github.com/python/cpython/issues/101283 is - # patched. If that is unpatched, then in the rare case the ComSpec environment - # variable is unset, the search for the shell itself is unsafe. Setting - # NoDefaultCurrentDirectoryInExePath in all cases, as is done here, is simpler and - # protects against that. (As above, the "1" can be any value.) + # When not using a shell, the current process does the search in a + # CreateProcessW API call, so the variable must be set in our environment. With + # a shell, this is unnecessary, in versions where + # https://github.com/python/cpython/issues/101283 is patched. If that is + # unpatched, then in the rare case the ComSpec environment variable is unset, + # the search for the shell itself is unsafe. Setting + # NoDefaultCurrentDirectoryInExePath in all cases, as is done here, is simpler + # and protects against that. (As above, the "1" can be any value.) with patch_env("NoDefaultCurrentDirectoryInExePath", "1"): return Popen( command, From b9d9e5642f84e7796b5242d155df7bdc28413303 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 03:19:28 -0400 Subject: [PATCH 0799/1392] Further improve _safer_popen_windows doc This adjusts the formatting of the _safer_popen_windows docstring and comments, and rewords some parts for clarity and brevity. --- git/cmd.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index d17c8836b..85398497f 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -264,22 +264,19 @@ def _safer_popen_windows( creationflags = subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP # When using a shell, the shell is the direct subprocess, so the variable must - # be set in its environment, to affect its search behavior. (The "1" can be any - # value.) + # be set in its environment, to affect its search behavior. if shell: - # The original may be immutable or reused by the caller. Make changes in a - # copy. + # The original may be immutable, or the caller may reuse it. Mutate a copy. env = {} if env is None else dict(env) - env["NoDefaultCurrentDirectoryInExePath"] = "1" + env["NoDefaultCurrentDirectoryInExePath"] = "1" # The "1" can be an value. # When not using a shell, the current process does the search in a # CreateProcessW API call, so the variable must be set in our environment. With - # a shell, this is unnecessary, in versions where - # https://github.com/python/cpython/issues/101283 is patched. If that is - # unpatched, then in the rare case the ComSpec environment variable is unset, - # the search for the shell itself is unsafe. Setting - # NoDefaultCurrentDirectoryInExePath in all cases, as is done here, is simpler - # and protects against that. (As above, the "1" can be any value.) + # a shell, that's unnecessary if https://github.com/python/cpython/issues/101283 + # is patched. In Python versions where it is unpatched, and in the rare case the + # ComSpec environment variable is unset, the search for the shell itself is + # unsafe. Setting NoDefaultCurrentDirectoryInExePath in all cases, as done here, + # is simpler and protects against that. (As above, the "1" can be any value.) with patch_env("NoDefaultCurrentDirectoryInExePath", "1"): return Popen( command, From 04a27531bf1d434a8da9d17298058d1f860fe019 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 03:36:32 -0400 Subject: [PATCH 0800/1392] Temporarily rename Commit_ish to Old_commit_ish And Lit_commit_ish to Lit_old_commit_ish. This is temporary, and only for purposes of bookkeeping while this type is split into two types. Specifically, Old_commit_ish will go away soon, because each occurrence of it will be changed to one of: - A newly redefined Commit_ish union of types representing only git object types that are sometimes commit-ish. - A newly introduced union (using some name formerly not used in GitPython) of all four types representing git object types, the same types as the old Commit_ish, here renamed Old_commit_ish, had overbroadly covered. - Perhaps in some cases something else. For context, see #1858 (including comments), and comments in #1859. --- git/index/base.py | 4 ++-- git/objects/base.py | 12 ++++++------ git/objects/submodule/base.py | 16 ++++++++-------- git/objects/submodule/root.py | 4 ++-- git/refs/head.py | 4 ++-- git/refs/reference.py | 4 ++-- git/refs/symbolic.py | 8 ++++---- git/refs/tag.py | 4 ++-- git/remote.py | 10 +++++----- git/repo/base.py | 8 ++++---- git/repo/fun.py | 12 ++++++------ git/types.py | 8 ++++---- 12 files changed, 47 insertions(+), 47 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 704235241..668607772 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -80,7 +80,7 @@ Union, ) -from git.types import Commit_ish, Literal, PathLike +from git.types import Old_commit_ish, Literal, PathLike if TYPE_CHECKING: from subprocess import Popen @@ -1127,7 +1127,7 @@ def move( def commit( self, message: str, - parent_commits: Union[Commit_ish, None] = None, + parent_commits: Union[Old_commit_ish, None] = None, head: bool = True, author: Union[None, "Actor"] = None, committer: Union[None, "Actor"] = None, diff --git a/git/objects/base.py b/git/objects/base.py index df9b2d9f1..dec07654a 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -16,7 +16,7 @@ from typing import Any, TYPE_CHECKING, Union -from git.types import PathLike, Commit_ish, Lit_commit_ish +from git.types import PathLike, Old_commit_ish, Lit_old_commit_ish if TYPE_CHECKING: from git.repo import Repo @@ -53,7 +53,7 @@ class Object(LazyMixin): * "tag object": https://git-scm.com/docs/gitglossary#def_tag_object :note: - See the :class:`~git.types.Commit_ish` union type. + See the :class:`~git.types.Old_commit_ish` union type. :note: :class:`~git.objects.submodule.base.Submodule` is defined under the hierarchy @@ -77,7 +77,7 @@ class Object(LazyMixin): __slots__ = ("repo", "binsha", "size") - type: Union[Lit_commit_ish, None] = None + type: Union[Lit_old_commit_ish, None] = None """String identifying (a concrete :class:`Object` subtype for) a git object type. The subtypes that this may name correspond to the kinds of git objects that exist, @@ -90,7 +90,7 @@ class Object(LazyMixin): ``None`` in concrete leaf subclasses representing specific git object types. :note: - See also :class:`~git.types.Commit_ish`. + See also :class:`~git.types.Old_commit_ish`. """ def __init__(self, repo: "Repo", binsha: bytes): @@ -113,7 +113,7 @@ def __init__(self, repo: "Repo", binsha: bytes): ) @classmethod - def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> Commit_ish: + def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> Old_commit_ish: """ :return: New :class:`Object` instance of a type appropriate to the object type behind @@ -130,7 +130,7 @@ def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> Commit_ish: return repo.rev_parse(str(id)) @classmethod - def new_from_sha(cls, repo: "Repo", sha1: bytes) -> Commit_ish: + def new_from_sha(cls, repo: "Repo", sha1: bytes) -> Old_commit_ish: """ :return: New object instance of a type appropriate to represent the given binary sha1 diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index a7d638c59..783990664 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -45,7 +45,7 @@ from typing import Callable, Dict, Mapping, Sequence, TYPE_CHECKING, cast from typing import Any, Iterator, Union -from git.types import Commit_ish, Literal, PathLike, TBD +from git.types import Old_commit_ish, Literal, PathLike, TBD if TYPE_CHECKING: from git.index import IndexFile @@ -112,7 +112,7 @@ def __init__( mode: Union[int, None] = None, path: Union[PathLike, None] = None, name: Union[str, None] = None, - parent_commit: Union[Commit_ish, None] = None, + parent_commit: Union[Old_commit_ish, None] = None, url: Union[str, None] = None, branch_path: Union[PathLike, None] = None, ) -> None: @@ -213,7 +213,7 @@ def __repr__(self) -> str: @classmethod def _config_parser( - cls, repo: "Repo", parent_commit: Union[Commit_ish, None], read_only: bool + cls, repo: "Repo", parent_commit: Union[Old_commit_ish, None], read_only: bool ) -> SubmoduleConfigParser: """ :return: @@ -264,7 +264,7 @@ def _clear_cache(self) -> None: # END for each name to delete @classmethod - def _sio_modules(cls, parent_commit: Commit_ish) -> BytesIO: + def _sio_modules(cls, parent_commit: Old_commit_ish) -> BytesIO: """ :return: Configuration file as :class:`~io.BytesIO` - we only access it through the @@ -277,7 +277,7 @@ def _sio_modules(cls, parent_commit: Commit_ish) -> BytesIO: def _config_parser_constrained(self, read_only: bool) -> SectionConstraint: """:return: Config parser constrained to our submodule in read or write mode""" try: - pc: Union["Commit_ish", None] = self.parent_commit + pc: Union["Old_commit_ish", None] = self.parent_commit except ValueError: pc = None # END handle empty parent repository @@ -1242,7 +1242,7 @@ def remove( return self - def set_parent_commit(self, commit: Union[Commit_ish, None], check: bool = True) -> "Submodule": + def set_parent_commit(self, commit: Union[Old_commit_ish, None], check: bool = True) -> "Submodule": """Set this instance to use the given commit whose tree is supposed to contain the ``.gitmodules`` blob. @@ -1495,7 +1495,7 @@ def url(self) -> str: return self._url @property - def parent_commit(self) -> "Commit_ish": + def parent_commit(self) -> "Old_commit_ish": """ :return: :class:`~git.objects.commit.Commit` instance with the tree containing the @@ -1557,7 +1557,7 @@ def children(self) -> IterableList["Submodule"]: def iter_items( cls, repo: "Repo", - parent_commit: Union[Commit_ish, str] = "HEAD", + parent_commit: Union[Old_commit_ish, str] = "HEAD", *Args: Any, **kwargs: Any, ) -> Iterator["Submodule"]: diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index d0d1818a9..8e697ee1d 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Union -from git.types import Commit_ish +from git.types import Old_commit_ish if TYPE_CHECKING: from git.repo import Repo @@ -77,7 +77,7 @@ def _clear_cache(self) -> None: def update( # type: ignore[override] self, - previous_commit: Union[Commit_ish, None] = None, + previous_commit: Union[Old_commit_ish, None] = None, recursive: bool = True, force_remove: bool = False, init: bool = True, diff --git a/git/refs/head.py b/git/refs/head.py index 4da614abd..e6bef598a 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -17,7 +17,7 @@ from typing import Any, Sequence, Union, TYPE_CHECKING -from git.types import PathLike, Commit_ish +from git.types import PathLike, Old_commit_ish if TYPE_CHECKING: from git.repo import Repo @@ -62,7 +62,7 @@ def orig_head(self) -> SymbolicReference: def reset( self, - commit: Union[Commit_ish, SymbolicReference, str] = "HEAD", + commit: Union[Old_commit_ish, SymbolicReference, str] = "HEAD", index: bool = True, working_tree: bool = False, paths: Union[PathLike, Sequence[PathLike], None] = None, diff --git a/git/refs/reference.py b/git/refs/reference.py index 2da511ce3..99366f710 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -11,7 +11,7 @@ # typing ------------------------------------------------------------------ from typing import Any, Callable, Iterator, Type, Union, TYPE_CHECKING -from git.types import Commit_ish, PathLike, _T +from git.types import Old_commit_ish, PathLike, _T if TYPE_CHECKING: from git.repo import Repo @@ -81,7 +81,7 @@ def __str__(self) -> str: # @ReservedAssignment def set_object( self, - object: Union[Commit_ish, "SymbolicReference", str], + object: Union[Old_commit_ish, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "Reference": """Special version which checks if the head-log needs an update as well. diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 142952e06..092824ff8 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -31,7 +31,7 @@ TYPE_CHECKING, cast, ) -from git.types import Commit_ish, PathLike +from git.types import Old_commit_ish, PathLike if TYPE_CHECKING: from git.repo import Repo @@ -278,7 +278,7 @@ def _get_ref_info(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> Union[T """ return cls._get_ref_info_helper(repo, ref_path) - def _get_object(self) -> Commit_ish: + def _get_object(self) -> Old_commit_ish: """ :return: The object our ref currently refers to. Refs can be cached, they will always @@ -345,7 +345,7 @@ def set_commit( def set_object( self, - object: Union[Commit_ish, "SymbolicReference", str], + object: Union[Old_commit_ish, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "SymbolicReference": """Set the object we point to, possibly dereference our symbolic reference @@ -404,7 +404,7 @@ def _get_reference(self) -> "SymbolicReference": def set_reference( self, - ref: Union[Commit_ish, "SymbolicReference", str], + ref: Union[Old_commit_ish, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "SymbolicReference": """Set ourselves to the given `ref`. diff --git a/git/refs/tag.py b/git/refs/tag.py index 6a6dad09a..0cf45cf20 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -15,7 +15,7 @@ # typing ------------------------------------------------------------------ from typing import Any, Type, Union, TYPE_CHECKING -from git.types import Commit_ish, PathLike +from git.types import Old_commit_ish, PathLike if TYPE_CHECKING: from git.repo import Repo @@ -82,7 +82,7 @@ def tag(self) -> Union["TagObject", None]: # Make object read-only. It should be reasonably hard to adjust an existing tag. @property - def object(self) -> Commit_ish: # type: ignore[override] + def object(self) -> Old_commit_ish: # type: ignore[override] return Reference._get_object(self) @classmethod diff --git a/git/remote.py b/git/remote.py index 2bd674199..cebfafb7b 100644 --- a/git/remote.py +++ b/git/remote.py @@ -41,7 +41,7 @@ overload, ) -from git.types import PathLike, Literal, Commit_ish +from git.types import PathLike, Literal, Old_commit_ish if TYPE_CHECKING: from git.repo.base import Repo @@ -196,7 +196,7 @@ def __init__( self.summary = summary @property - def old_commit(self) -> Union[str, SymbolicReference, Commit_ish, None]: + def old_commit(self) -> Union[str, SymbolicReference, Old_commit_ish, None]: return self._old_commit_sha and self._remote.repo.commit(self._old_commit_sha) or None @property @@ -362,7 +362,7 @@ def __init__( ref: SymbolicReference, flags: int, note: str = "", - old_commit: Union[Commit_ish, None] = None, + old_commit: Union[Old_commit_ish, None] = None, remote_ref_path: Optional[PathLike] = None, ) -> None: """Initialize a new instance.""" @@ -381,7 +381,7 @@ def name(self) -> str: return self.ref.name @property - def commit(self) -> Commit_ish: + def commit(self) -> Old_commit_ish: """:return: Commit of our remote ref""" return self.ref.commit @@ -438,7 +438,7 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": # Parse operation string for more info. # This makes no sense for symbolic refs, but we parse it anyway. - old_commit: Union[Commit_ish, None] = None + old_commit: Union[Old_commit_ish, None] = None is_tag_operation = False if "rejected" in operation: flags |= cls.REJECTED diff --git a/git/repo/base.py b/git/repo/base.py index 9b42ec85a..62f458f68 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -55,7 +55,7 @@ TBD, PathLike, Lit_config_levels, - Commit_ish, + Old_commit_ish, CallableProgress, Tree_ish, assert_never, @@ -696,7 +696,7 @@ def config_writer(self, config_level: Lit_config_levels = "repository") -> GitCo """ return GitConfigParser(self._get_config_path(config_level), read_only=False, repo=self, merge_includes=False) - def commit(self, rev: Union[str, Commit_ish, None] = None) -> Commit: + def commit(self, rev: Union[str, Old_commit_ish, None] = None) -> Commit: """The :class:`~git.objects.commit.Commit` object for the specified revision. :param rev: @@ -772,7 +772,7 @@ def iter_commits( return Commit.iter_items(self, rev, paths, **kwargs) - def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: + def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Old_commit_ish, None]]: R"""Find the closest common ancestor for the given revision (:class:`~git.objects.commit.Commit`\s, :class:`~git.refs.tag.Tag`\s, :class:`~git.refs.reference.Reference`\s, etc.). @@ -797,7 +797,7 @@ def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Commit_ish, None]]: raise ValueError("Please specify at least two revs, got only %i" % len(rev)) # END handle input - res: List[Union[Commit_ish, None]] = [] + res: List[Union[Old_commit_ish, None]] = [] try: lines = self.git.merge_base(*rev, **kwargs).splitlines() # List[str] except GitCommandError as err: diff --git a/git/repo/fun.py b/git/repo/fun.py index e3c69c68c..6bc998bb7 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -25,7 +25,7 @@ # Typing ---------------------------------------------------------------------- from typing import Union, Optional, cast, TYPE_CHECKING -from git.types import Commit_ish +from git.types import Old_commit_ish if TYPE_CHECKING: from git.types import PathLike @@ -249,7 +249,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: raise NotImplementedError("commit by message search (regex)") # END handle search - obj: Union[Commit_ish, "Reference", None] = None + obj: Union[Old_commit_ish, "Reference", None] = None ref = None output_type = "commit" start = 0 @@ -271,7 +271,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: if token == "@": ref = cast("Reference", name_to_object(repo, rev[:start], return_ref=True)) else: - obj = cast(Commit_ish, name_to_object(repo, rev[:start])) + obj = cast(Old_commit_ish, name_to_object(repo, rev[:start])) # END handle token # END handle refname else: @@ -296,7 +296,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: pass # Default. elif output_type == "tree": try: - obj = cast(Commit_ish, obj) + obj = cast(Old_commit_ish, obj) obj = to_commit(obj).tree except (AttributeError, ValueError): pass # Error raised later. @@ -369,7 +369,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: parsed_to = start # Handle hierarchy walk. try: - obj = cast(Commit_ish, obj) + obj = cast(Old_commit_ish, obj) if token == "~": obj = to_commit(obj) for _ in range(num): @@ -398,7 +398,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: # Still no obj? It's probably a simple name. if obj is None: - obj = cast(Commit_ish, name_to_object(repo, rev)) + obj = cast(Old_commit_ish, name_to_object(repo, rev)) parsed_to = lr # END handle simple name diff --git a/git/types.py b/git/types.py index 8f71d079a..e9ef48ace 100644 --- a/git/types.py +++ b/git/types.py @@ -61,7 +61,7 @@ commit), is not covered as part of this union type. """ -Commit_ish = Union["Commit", "TagObject", "Blob", "Tree"] +Old_commit_ish = Union["Commit", "TagObject", "Blob", "Tree"] """Union of the :class:`~git.objects.base.Object`-based types that represent git object types. This union is often usable where a commit-ish is expected, but is not actually limited to types representing commit-ish git objects. @@ -86,18 +86,18 @@ compatibility. """ -Lit_commit_ish = Literal["commit", "tag", "blob", "tree"] +Lit_old_commit_ish = Literal["commit", "tag", "blob", "tree"] """Literal strings identifying concrete :class:`~git.objects.base.Object` subtypes representing git object types. See the :class:`Object.type ` attribute. :note: - See also :class:`Commit_ish`, a union of the the :class:`~git.objects.base.Object` + See also :class:`Old_commit_ish`, a union of the the :class:`~git.objects.base.Object` subtypes associated with these literal strings. :note: - As noted in :class:`Commit_ish`, this is not limited to types of git objects that + As noted in :class:`Old_commit_ish`, this is not limited to types of git objects that are actually commit-ish. """ From 787f65cc7d80cd4fb73b37847718a32473b5d985 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 12:44:37 -0400 Subject: [PATCH 0801/1392] Define and document AnyGitObject and (new) Commit_ish These provide for the changes planned in 04a2753 before this (at least the first two cases listed). But annotations still need to be changed to use these types. --- git/types.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/git/types.py b/git/types.py index e9ef48ace..1ef2b1523 100644 --- a/git/types.py +++ b/git/types.py @@ -46,6 +46,33 @@ _T = TypeVar("_T") """Type variable used internally in GitPython.""" +AnyGitObject = Union["Commit", "Tree", "TagObject", "Blob"] +"""Union of the :class:`~git.objects.base.Object`-based types that represent actual git +object types. + +As noted in :class:`~git.objects.base.Object`, which has further details, these are: + +* :class:`Blob ` +* :class:`Tree ` +* :class:`Commit ` +* :class:`TagObject ` + +Those GitPython classes represent the four git object types, per gitglossary(7): + +* "blob": https://git-scm.com/docs/gitglossary#def_blob_object +* "tree object": https://git-scm.com/docs/gitglossary#def_tree_object +* "commit object": https://git-scm.com/docs/gitglossary#def_commit_object +* "tag object": https://git-scm.com/docs/gitglossary#def_tag_object + +For more general information on git objects and their types as git understands them: + +* "object": https://git-scm.com/docs/gitglossary#def_object +* "object type": https://git-scm.com/docs/gitglossary#def_object_type + +:note: + See also the :class:`Tree_ish` and :class:`Commit_ish` unions. +""" + Tree_ish = Union["Commit", "Tree"] """Union of :class:`~git.objects.base.Object`-based types that are inherently tree-ish. @@ -54,13 +81,40 @@ :note: This union comprises **only** the :class:`~git.objects.commit.Commit` and :class:`~git.objects.tree.Tree` classes, **all** of whose instances are tree-ish. - This is done because of the way GitPython uses it as a static type annotation. + This has been done because of the way GitPython uses it as a static type annotation. :class:`~git.objects.tag.TagObject`, some but not all of whose instances are tree-ish (those representing git tag objects that ultimately resolve to a tree or commit), is not covered as part of this union type. + +:note: + See also the :class:`AnyGitObject` union of all four classes corresponding to git + object types. +""" + +Commit_ish = Union["Commit", "TagObject"] +"""Union of :class:`~git.objects.base.Object`-based types that are sometimes commit-ish. + +See gitglossary(7) on "commit-ish": https://git-scm.com/docs/gitglossary#def_commit-ish + +:note: + :class:`~git.objects.commit.Commit` is the only class whose instances are all + commit-ish. This union type includes :class:`~git.objects.commit.Commit`, but also + :class:`~git.objects.tag.TagObject`, only **some** of whose instances are + commit-ish. Whether a particular :class:`~git.objects.tag.TagObject` peels + (recursively dereferences) to a commit can in general only be known at runtime. + +:note: + This is an inversion of the situation with :class:`Tree_ish`. This union is broader + than all commit-ish objects, while :class:`Tree_ish` is narrower than all tree-ish + objects. + +:note: + See also the :class:`AnyGitObject` union of all four classes corresponding to git + object types. """ +# FIXME: Replace uses with AnyGitObject and Commit_ish, and remove this. Old_commit_ish = Union["Commit", "TagObject", "Blob", "Tree"] """Union of the :class:`~git.objects.base.Object`-based types that represent git object types. This union is often usable where a commit-ish is expected, but is not actually From 1fe4dc88dcab2a61608c247853a502f7c5186d4b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 14:09:13 -0400 Subject: [PATCH 0802/1392] Define GitObjectTypeString and update Object to use it This is equivalent to the old Lit_commit_ish union; it is the type of a literal string that has one of the four values that represent actual git object types. The old Lit_commit_ish union was only used in one place in GitPython: to annotate Object.type. This replaces that and updates its docstring, as well as the Object class docstring. (One change in the Object class docstring--adding a mention of the RootModule subclass of Submodule--is not conceptually related to these other changes.) --- git/objects/base.py | 14 ++++++++------ git/types.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/git/objects/base.py b/git/objects/base.py index dec07654a..3b2514e9b 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -16,7 +16,7 @@ from typing import Any, TYPE_CHECKING, Union -from git.types import PathLike, Old_commit_ish, Lit_old_commit_ish +from git.types import GitObjectTypeString, Old_commit_ish, PathLike if TYPE_CHECKING: from git.repo import Repo @@ -36,7 +36,7 @@ class Object(LazyMixin): """Base class for classes representing git object types. - The following leaf classes represent specific kinds of git objects: + The following four leaf classes represent specific kinds of git objects: * :class:`Blob ` * :class:`Tree ` @@ -53,12 +53,14 @@ class Object(LazyMixin): * "tag object": https://git-scm.com/docs/gitglossary#def_tag_object :note: - See the :class:`~git.types.Old_commit_ish` union type. + See the :class:`~git.types.AnyGitObject` union type of the four leaf subclasses + that represent actual git object types. :note: :class:`~git.objects.submodule.base.Submodule` is defined under the hierarchy rooted at this :class:`Object` class, even though submodules are not really a - type of git object. + type of git object. (This also applies to its + :class:`~git.objects.submodule.root.RootModule` subclass.) :note: This :class:`Object` class should not be confused with :class:`object` (the root @@ -77,7 +79,7 @@ class Object(LazyMixin): __slots__ = ("repo", "binsha", "size") - type: Union[Lit_old_commit_ish, None] = None + type: Union[GitObjectTypeString, None] = None """String identifying (a concrete :class:`Object` subtype for) a git object type. The subtypes that this may name correspond to the kinds of git objects that exist, @@ -90,7 +92,7 @@ class Object(LazyMixin): ``None`` in concrete leaf subclasses representing specific git object types. :note: - See also :class:`~git.types.Old_commit_ish`. + See also :class:`~git.types.GitObjectTypeString`. """ def __init__(self, repo: "Repo", binsha: bytes): diff --git a/git/types.py b/git/types.py index 1ef2b1523..95a5816f3 100644 --- a/git/types.py +++ b/git/types.py @@ -114,6 +114,18 @@ object types. """ +GitObjectTypeString = Literal["commit", "tag", "blob", "tree"] +"""Literal strings identifying git object types and the +:class:`~git.objects.base.Object`-based types that represent them. + +See the :attr:`Object.type ` attribute. These are its +values in :class:`~git.objects.base.Object` subclasses that represent git objects. These +literals therefore correspond to the types in the :class:`AnyGitObject` union. + +These are the same strings git itself uses to identify its four object types. See +gitglossary(7) on "object type": https://git-scm.com/docs/gitglossary#def_object_type +""" + # FIXME: Replace uses with AnyGitObject and Commit_ish, and remove this. Old_commit_ish = Union["Commit", "TagObject", "Blob", "Tree"] """Union of the :class:`~git.objects.base.Object`-based types that represent git object @@ -140,6 +152,8 @@ compatibility. """ +# FIXME: After replacing the one use with GitObjectTypeString, define Lit_commit_ish +# somehow (it is a breaking change to remove it entirely). Maybe deprecate it. Lit_old_commit_ish = Literal["commit", "tag", "blob", "tree"] """Literal strings identifying concrete :class:`~git.objects.base.Object` subtypes representing git object types. From 7328a00170c20425760b0c20212042c1b884cb5b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 16:51:01 -0400 Subject: [PATCH 0803/1392] Start fixing annotations that used the old Commit_ish --- git/objects/base.py | 18 +++++++++------- git/refs/symbolic.py | 50 +++++++++++++++++++++++--------------------- git/refs/tag.py | 9 ++++---- git/remote.py | 8 +++---- git/repo/base.py | 38 ++++++++++++++++----------------- 5 files changed, 63 insertions(+), 60 deletions(-) diff --git a/git/objects/base.py b/git/objects/base.py index 3b2514e9b..90e20c2b6 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -3,12 +3,12 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from git.exc import WorkTreeRepositoryUnsupported -from git.util import LazyMixin, join_path_native, stream_copy, bin_to_hex - import gitdb.typ as dbtyp import os.path as osp +from git.exc import WorkTreeRepositoryUnsupported +from git.util import LazyMixin, join_path_native, stream_copy, bin_to_hex + from .util import get_object_type_by_name @@ -16,15 +16,17 @@ from typing import Any, TYPE_CHECKING, Union -from git.types import GitObjectTypeString, Old_commit_ish, PathLike +from git.types import AnyGitObject, GitObjectTypeString, PathLike if TYPE_CHECKING: - from git.repo import Repo from gitdb.base import OStream + + from git.refs.reference import Reference + from git.repo import Repo + from .tree import Tree from .blob import Blob from .submodule.base import Submodule - from git.refs.reference import Reference IndexObjUnion = Union["Tree", "Blob", "Submodule"] @@ -115,7 +117,7 @@ def __init__(self, repo: "Repo", binsha: bytes): ) @classmethod - def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> Old_commit_ish: + def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> AnyGitObject: """ :return: New :class:`Object` instance of a type appropriate to the object type behind @@ -132,7 +134,7 @@ def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> Old_commit_ish: return repo.rev_parse(str(id)) @classmethod - def new_from_sha(cls, repo: "Repo", sha1: bytes) -> Old_commit_ish: + def new_from_sha(cls, repo: "Repo", sha1: bytes) -> AnyGitObject: """ :return: New object instance of a type appropriate to represent the given binary sha1 diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 092824ff8..4a39875a7 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -6,17 +6,16 @@ from git.compat import defenc from git.objects import Object from git.objects.commit import Commit +from git.refs.log import RefLog from git.util import ( + LockedFD, + assure_directory_exists, + hex_to_bin, join_path, join_path_native, to_native_path_linux, - assure_directory_exists, - hex_to_bin, - LockedFD, ) -from gitdb.exc import BadObject, BadName - -from .log import RefLog +from gitdb.exc import BadName, BadObject # typing ------------------------------------------------------------------ @@ -24,21 +23,21 @@ Any, Iterator, List, + TYPE_CHECKING, Tuple, Type, TypeVar, Union, - TYPE_CHECKING, cast, ) -from git.types import Old_commit_ish, PathLike +from git.types import AnyGitObject, PathLike if TYPE_CHECKING: - from git.repo import Repo - from git.refs import Head, TagReference, RemoteReference, Reference - from .log import RefLogEntry from git.config import GitConfigParser from git.objects.commit import Actor + from git.refs import Head, TagReference, RemoteReference, Reference + from git.refs.log import RefLogEntry + from git.repo import Repo T_References = TypeVar("T_References", bound="SymbolicReference") @@ -278,7 +277,7 @@ def _get_ref_info(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> Union[T """ return cls._get_ref_info_helper(repo, ref_path) - def _get_object(self) -> Old_commit_ish: + def _get_object(self) -> AnyGitObject: """ :return: The object our ref currently refers to. Refs can be cached, they will always @@ -345,7 +344,7 @@ def set_commit( def set_object( self, - object: Union[Old_commit_ish, "SymbolicReference", str], + object: Union[AnyGitObject, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "SymbolicReference": """Set the object we point to, possibly dereference our symbolic reference @@ -353,9 +352,12 @@ def set_object( :param object: A refspec, a :class:`SymbolicReference` or an - :class:`~git.objects.base.Object` instance. :class:`SymbolicReference` - instances will be dereferenced beforehand to obtain the object they point - to. + :class:`~git.objects.base.Object` instance. + + * :class:`SymbolicReference` instances will be dereferenced beforehand to + obtain the git object they point to. + * :class:`~git.objects.base.Object` instances must represent git objects + (:class:`~git.types.AnyGitObject`). :param logmsg: If not ``None``, the message will be used in the reflog entry to be written. @@ -404,22 +406,22 @@ def _get_reference(self) -> "SymbolicReference": def set_reference( self, - ref: Union[Old_commit_ish, "SymbolicReference", str], + ref: Union[AnyGitObject, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "SymbolicReference": """Set ourselves to the given `ref`. - It will stay a symbol if the ref is a :class:`~git.refs.reference.Reference`. + It will stay a symbol if the `ref` is a :class:`~git.refs.reference.Reference`. - Otherwise an Object, given as :class:`~git.objects.base.Object` instance or - refspec, is assumed and if valid, will be set which effectively detaches the - reference if it was a purely symbolic one. + Otherwise a git object, specified as a :class:`~git.objects.base.Object` + instance or refspec, is assumed. If it is valid, this reference will be set to + it, which effectively detaches the reference if it was a purely symbolic one. :param ref: A :class:`SymbolicReference` instance, an :class:`~git.objects.base.Object` - instance, or a refspec string. Only if the ref is a - :class:`SymbolicReference` instance, we will point to it. Everything else is - dereferenced to obtain the actual object. + instance (specifically an :class:`~git.types.AnyGitObject`), or a refspec + string. Only if the ref is a :class:`SymbolicReference` instance, we will + point to it. Everything else is dereferenced to obtain the actual object. :param logmsg: If set to a string, the message will be used in the reflog. diff --git a/git/refs/tag.py b/git/refs/tag.py index 0cf45cf20..a1d0b470f 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -14,14 +14,15 @@ # typing ------------------------------------------------------------------ -from typing import Any, Type, Union, TYPE_CHECKING -from git.types import Old_commit_ish, PathLike +from typing import Any, TYPE_CHECKING, Type, Union + +from git.types import AnyGitObject, PathLike if TYPE_CHECKING: - from git.repo import Repo from git.objects import Commit from git.objects import TagObject from git.refs import SymbolicReference + from git.repo import Repo # ------------------------------------------------------------------------------ @@ -82,7 +83,7 @@ def tag(self) -> Union["TagObject", None]: # Make object read-only. It should be reasonably hard to adjust an existing tag. @property - def object(self) -> Old_commit_ish: # type: ignore[override] + def object(self) -> AnyGitObject: # type: ignore[override] return Reference._get_object(self) @classmethod diff --git a/git/remote.py b/git/remote.py index cebfafb7b..01fe3771d 100644 --- a/git/remote.py +++ b/git/remote.py @@ -44,11 +44,9 @@ from git.types import PathLike, Literal, Old_commit_ish if TYPE_CHECKING: - from git.repo.base import Repo + from git.objects.commit import Commit from git.objects.submodule.base import UpdateProgress - - # from git.objects.commit import Commit - # from git.objects import Blob, Tree, TagObject + from git.repo.base import Repo flagKeyLiteral = Literal[" ", "!", "+", "-", "*", "=", "t", "?"] @@ -381,7 +379,7 @@ def name(self) -> str: return self.ref.name @property - def commit(self) -> Old_commit_ish: + def commit(self) -> "Commit": """:return: Commit of our remote ref""" return self.ref.commit diff --git a/git/repo/base.py b/git/repo/base.py index 62f458f68..5e7645366 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -34,29 +34,29 @@ from git.remote import Remote, add_progress, to_progress_instance from git.util import ( Actor, - finalize_process, cygpath, - hex_to_bin, expand_path, + finalize_process, + hex_to_bin, remove_password_if_present, ) from .fun import ( - rev_parse, - is_git_dir, find_submodule_git_dir, - touch, find_worktree_git_dir, + is_git_dir, + rev_parse, + touch, ) # typing ------------------------------------------------------ from git.types import ( - TBD, - PathLike, - Lit_config_levels, - Old_commit_ish, CallableProgress, + Commit_ish, + Lit_config_levels, + PathLike, + TBD, Tree_ish, assert_never, ) @@ -68,25 +68,25 @@ Iterator, List, Mapping, + NamedTuple, Optional, Sequence, + TYPE_CHECKING, TextIO, Tuple, Type, Union, - NamedTuple, cast, - TYPE_CHECKING, ) from git.types import ConfigLevels_Tup, TypedDict if TYPE_CHECKING: - from git.util import IterableList - from git.refs.symbolic import SymbolicReference from git.objects import Tree from git.objects.submodule.base import UpdateProgress + from git.refs.symbolic import SymbolicReference from git.remote import RemoteProgress + from git.util import IterableList # ----------------------------------------------------------- @@ -96,7 +96,7 @@ class BlameEntry(NamedTuple): - commit: Dict[str, "Commit"] + commit: Dict[str, Commit] linenos: range orig_path: Optional[str] orig_linenos: range @@ -696,7 +696,7 @@ def config_writer(self, config_level: Lit_config_levels = "repository") -> GitCo """ return GitConfigParser(self._get_config_path(config_level), read_only=False, repo=self, merge_includes=False) - def commit(self, rev: Union[str, Old_commit_ish, None] = None) -> Commit: + def commit(self, rev: Union[str, Commit_ish, None] = None) -> Commit: """The :class:`~git.objects.commit.Commit` object for the specified revision. :param rev: @@ -772,7 +772,7 @@ def iter_commits( return Commit.iter_items(self, rev, paths, **kwargs) - def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Old_commit_ish, None]]: + def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Commit]: R"""Find the closest common ancestor for the given revision (:class:`~git.objects.commit.Commit`\s, :class:`~git.refs.tag.Tag`\s, :class:`~git.refs.reference.Reference`\s, etc.). @@ -797,9 +797,9 @@ def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Old_commit_ish, Non raise ValueError("Please specify at least two revs, got only %i" % len(rev)) # END handle input - res: List[Union[Old_commit_ish, None]] = [] + res: List[Commit] = [] try: - lines = self.git.merge_base(*rev, **kwargs).splitlines() # List[str] + lines: List[str] = self.git.merge_base(*rev, **kwargs).splitlines() except GitCommandError as err: if err.status == 128: raise @@ -815,7 +815,7 @@ def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Union[Old_commit_ish, Non return res - def is_ancestor(self, ancestor_rev: "Commit", rev: "Commit") -> bool: + def is_ancestor(self, ancestor_rev: Commit, rev: Commit) -> bool: """Check if a commit is an ancestor of another. :param ancestor_rev: From 191f4cf2e7fe2aef2e8583910b50b878bc66d3ed Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 18:50:28 -0400 Subject: [PATCH 0804/1392] Fix some annotations in git.repo.fun --- git/repo/fun.py | 50 ++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/git/repo/fun.py b/git/repo/fun.py index 6bc998bb7..ad8a9fd90 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -6,34 +6,30 @@ from __future__ import annotations import os -import stat +import os.path as osp from pathlib import Path +import stat from string import digits +from git.cmd import Git from git.exc import WorkTreeRepositoryUnsupported from git.objects import Object from git.refs import SymbolicReference from git.util import hex_to_bin, bin_to_hex, cygpath -from gitdb.exc import ( - BadObject, - BadName, -) - -import os.path as osp -from git.cmd import Git +from gitdb.exc import BadName, BadObject # Typing ---------------------------------------------------------------------- -from typing import Union, Optional, cast, TYPE_CHECKING -from git.types import Old_commit_ish +from typing import Optional, TYPE_CHECKING, Union, cast, overload + +from git.types import AnyGitObject, Literal, Old_commit_ish, PathLike if TYPE_CHECKING: - from git.types import PathLike - from .base import Repo from git.db import GitCmdObjectDB - from git.refs.reference import Reference from git.objects import Commit, TagObject, Blob, Tree + from git.refs.reference import Reference from git.refs.tag import Tag + from .base import Repo # ---------------------------------------------------------------------------- @@ -56,7 +52,7 @@ def touch(filename: str) -> str: return filename -def is_git_dir(d: "PathLike") -> bool: +def is_git_dir(d: PathLike) -> bool: """This is taken from the git setup.c:is_git_directory function. :raise git.exc.WorkTreeRepositoryUnsupported: @@ -79,7 +75,7 @@ def is_git_dir(d: "PathLike") -> bool: return False -def find_worktree_git_dir(dotgit: "PathLike") -> Optional[str]: +def find_worktree_git_dir(dotgit: PathLike) -> Optional[str]: """Search for a gitdir for this worktree.""" try: statbuf = os.stat(dotgit) @@ -98,7 +94,7 @@ def find_worktree_git_dir(dotgit: "PathLike") -> Optional[str]: return None -def find_submodule_git_dir(d: "PathLike") -> Optional["PathLike"]: +def find_submodule_git_dir(d: PathLike) -> Optional[PathLike]: """Search for a submodule repo.""" if is_git_dir(d): return d @@ -141,9 +137,17 @@ def short_to_long(odb: "GitCmdObjectDB", hexsha: str) -> Optional[bytes]: # END exception handling -def name_to_object( - repo: "Repo", name: str, return_ref: bool = False -) -> Union[SymbolicReference, "Commit", "TagObject", "Blob", "Tree"]: +@overload +def name_to_object(repo: "Repo", name: str, return_ref: Literal[False] = ...) -> AnyGitObject: + ... + + +@overload +def name_to_object(repo: "Repo", name: str, return_ref: Literal[True]) -> Union[AnyGitObject, SymbolicReference]: + ... + + +def name_to_object(repo: "Repo", name: str, return_ref: bool = False) -> Union[AnyGitObject, SymbolicReference]: """ :return: Object specified by the given name - hexshas (short and long) as well as @@ -151,8 +155,8 @@ def name_to_object( :param return_ref: If ``True``, and name specifies a reference, we will return the reference - instead of the object. Otherwise it will raise `~gitdb.exc.BadObject` or - `~gitdb.exc.BadName`. + instead of the object. Otherwise it will raise :class:`~gitdb.exc.BadObject` or + :class:`~gitdb.exc.BadName`. """ hexsha: Union[None, str, bytes] = None @@ -201,7 +205,7 @@ def name_to_object( return Object.new_from_sha(repo, hex_to_bin(hexsha)) -def deref_tag(tag: "Tag") -> "TagObject": +def deref_tag(tag: "Tag") -> AnyGitObject: """Recursively dereference a tag and return the resulting object.""" while True: try: @@ -212,7 +216,7 @@ def deref_tag(tag: "Tag") -> "TagObject": return tag -def to_commit(obj: Object) -> Union["Commit", "TagObject"]: +def to_commit(obj: Object) -> "Commit": """Convert the given object to a commit if possible and return it.""" if obj.type == "tag": obj = deref_tag(obj) From d1ce94048ecaccf64f6bee0ad20f33df256e1139 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 18:53:06 -0400 Subject: [PATCH 0805/1392] Remove extra `parents` param in Commit.__init__ docstring It was listed earlier and was incorrect, not corresponding to the position or type of this initializer's actual `parents` parameter. --- git/objects/commit.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 28df86c9d..c5460b07b 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -112,15 +112,12 @@ def __init__( encoding: Union[str, None] = None, gpgsig: Union[str, None] = None, ) -> None: - R"""Instantiate a new :class:`Commit`. All keyword arguments taking ``None`` as + """Instantiate a new :class:`Commit`. All keyword arguments taking ``None`` as default will be implicitly set on first query. :param binsha: 20 byte sha1. - :param parents: tuple(Commit, ...) - A tuple of commit ids or actual :class:`Commit`\s. - :param tree: A :class:`~git.objects.tree.Tree` object. From fe42ca78c4dfb8a9afb442307d43c380c9751a71 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 21:12:22 -0400 Subject: [PATCH 0806/1392] Help tools know the type of a Commit's `parents` mypy seems not to need this, and already infers the specific type List[Commit], but some other type checkers -- at least pylance, and thus probably also pyright -- do not infer this. --- git/objects/commit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index c5460b07b..3dec9a14d 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -753,7 +753,7 @@ def _deserialize(self, stream: BytesIO) -> "Commit": readline = stream.readline self.tree = Tree(self.repo, hex_to_bin(readline().split()[1]), Tree.tree_id << 12, "") - self.parents = [] + self.parents = [] # type: List[Commit] next_line = None while True: parent_line = readline() From e66297a104c7acfcac21dd0a33e7d0581596f251 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 21:17:00 -0400 Subject: [PATCH 0807/1392] Keep the type of a Commit's `parents` from being too narrow The type should be Sequence[Commit], rather than List[Commit] as some type checkers seem already to have been inferring and as some others were caused to infer by the indirect fix that precedes this. --- git/objects/commit.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 3dec9a14d..6d3864be6 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -79,8 +79,8 @@ class Commit(base.Object, TraversableIterableObj, Diffable, Serializable): # INVARIANTS default_encoding = "UTF-8" - # object configuration type: Literal["commit"] = "commit" + __slots__ = ( "tree", "author", @@ -94,8 +94,11 @@ class Commit(base.Object, TraversableIterableObj, Diffable, Serializable): "encoding", "gpgsig", ) + _id_attribute_ = "hexsha" + parents: Sequence["Commit"] + def __init__( self, repo: "Repo", @@ -753,7 +756,7 @@ def _deserialize(self, stream: BytesIO) -> "Commit": readline = stream.readline self.tree = Tree(self.repo, hex_to_bin(readline().split()[1]), Tree.tree_id << 12, "") - self.parents = [] # type: List[Commit] + self.parents = [] next_line = None while True: parent_line = readline() From fe7f9f2b27fa43d0340887abce63820dd40ea607 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 21:25:48 -0400 Subject: [PATCH 0808/1392] Fix remaining old Commit_ish annotations in git.repo.fun Some were replaced in 191f4cf (which also included other fixes). The fixes here consist mostly of replacing the remaining uses of the old Commit_ish, though some now-unneeded casting is also removed. --- git/repo/fun.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/git/repo/fun.py b/git/repo/fun.py index ad8a9fd90..b65432b09 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -22,11 +22,11 @@ from typing import Optional, TYPE_CHECKING, Union, cast, overload -from git.types import AnyGitObject, Literal, Old_commit_ish, PathLike +from git.types import AnyGitObject, Literal, PathLike if TYPE_CHECKING: from git.db import GitCmdObjectDB - from git.objects import Commit, TagObject, Blob, Tree + from git.objects import Commit, TagObject from git.refs.reference import Reference from git.refs.tag import Tag from .base import Repo @@ -227,12 +227,18 @@ def to_commit(obj: Object) -> "Commit": return obj -def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: - """ +def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: + """Parse a revision string. Like ``git rev-parse``. + :return: - `~git.objects.base.Object` at the given revision, either - `~git.objects.commit.Commit`, `~git.refs.tag.Tag`, `~git.objects.tree.Tree` or - `~git.objects.blob.Blob`. + `~git.objects.base.Object` at the given revision. + + This may be any type of git object: + + * :class:`Commit ` + * :class:`TagObject ` + * :class:`Tree ` + * :class:`Blob ` :param rev: ``git rev-parse``-compatible revision specification as string. Please see @@ -253,7 +259,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: raise NotImplementedError("commit by message search (regex)") # END handle search - obj: Union[Old_commit_ish, "Reference", None] = None + obj: Optional[AnyGitObject] = None ref = None output_type = "commit" start = 0 @@ -275,12 +281,10 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: if token == "@": ref = cast("Reference", name_to_object(repo, rev[:start], return_ref=True)) else: - obj = cast(Old_commit_ish, name_to_object(repo, rev[:start])) + obj = name_to_object(repo, rev[:start]) # END handle token # END handle refname else: - assert obj is not None - if ref is not None: obj = cast("Commit", ref.commit) # END handle ref @@ -300,7 +304,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: pass # Default. elif output_type == "tree": try: - obj = cast(Old_commit_ish, obj) + obj = cast(AnyGitObject, obj) obj = to_commit(obj).tree except (AttributeError, ValueError): pass # Error raised later. @@ -373,7 +377,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: parsed_to = start # Handle hierarchy walk. try: - obj = cast(Old_commit_ish, obj) + obj = cast(AnyGitObject, obj) if token == "~": obj = to_commit(obj) for _ in range(num): @@ -402,7 +406,7 @@ def rev_parse(repo: "Repo", rev: str) -> Union["Commit", "Tag", "Tree", "Blob"]: # Still no obj? It's probably a simple name. if obj is None: - obj = cast(Old_commit_ish, name_to_object(repo, rev)) + obj = name_to_object(repo, rev) parsed_to = lr # END handle simple name From ab2782762515a8e49acc1f195bf8d8aebbafff6c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 22:12:37 -0400 Subject: [PATCH 0809/1392] Fix remaining old Commit_ish annotations in git.refs (This also improves the sorting of imports where they are already being changed, and fixes a typo in a comment.) --- git/refs/head.py | 10 +++++----- git/refs/reference.py | 14 +++++--------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/git/refs/head.py b/git/refs/head.py index e6bef598a..4eff7abb8 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -15,14 +15,14 @@ # typing --------------------------------------------------- -from typing import Any, Sequence, Union, TYPE_CHECKING +from typing import Any, Sequence, TYPE_CHECKING, Union -from git.types import PathLike, Old_commit_ish +from git.types import Commit_ish, PathLike if TYPE_CHECKING: - from git.repo import Repo from git.objects import Commit from git.refs import RemoteReference + from git.repo import Repo # ------------------------------------------------------------------- @@ -62,7 +62,7 @@ def orig_head(self) -> SymbolicReference: def reset( self, - commit: Union[Old_commit_ish, SymbolicReference, str] = "HEAD", + commit: Union[Commit_ish, SymbolicReference, str] = "HEAD", index: bool = True, working_tree: bool = False, paths: Union[PathLike, Sequence[PathLike], None] = None, @@ -99,7 +99,7 @@ def reset( if index: mode = "--mixed" - # Tt appears some git versions declare mixed and paths deprecated. + # It appears some git versions declare mixed and paths deprecated. # See http://github.com/Byron/GitPython/issues#issue/2. if paths: mode = None diff --git a/git/refs/reference.py b/git/refs/reference.py index 99366f710..cf418aa5d 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -1,24 +1,20 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from git.util import ( - LazyMixin, - IterableObj, -) +from git.util import IterableObj, LazyMixin from .symbolic import SymbolicReference, T_References - # typing ------------------------------------------------------------------ -from typing import Any, Callable, Iterator, Type, Union, TYPE_CHECKING -from git.types import Old_commit_ish, PathLike, _T +from typing import Any, Callable, Iterator, TYPE_CHECKING, Type, Union + +from git.types import AnyGitObject, PathLike, _T if TYPE_CHECKING: from git.repo import Repo # ------------------------------------------------------------------------------ - __all__ = ["Reference"] # { Utilities @@ -81,7 +77,7 @@ def __str__(self) -> str: # @ReservedAssignment def set_object( self, - object: Union[Old_commit_ish, "SymbolicReference", str], + object: Union[AnyGitObject, "SymbolicReference", str], logmsg: Union[str, None] = None, ) -> "Reference": """Special version which checks if the head-log needs an update as well. From b4b6e1ee0c94dd22886ee326f476f8dcfd49647b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 22:32:12 -0400 Subject: [PATCH 0810/1392] Fix IndexFile.commit `parent_commits` annotation This changes that parameter's annotation to be equivalent to the annotation of the corresponding Commit.create_from_tree parameter. It had been annotated as the old Commit_ish type, but it passes that argument to Commit.create_from_tree, and it also refers to that method in its docstring for usage. Commit.create_from_tree annotates this argument as an optional list of Commit objects, and if the objects obtained when it iterates them are not Commit instances, then an exception is raised. Even if being able to pass other commit-ish things (or references, strings, etc.) is desirable, that does not currently work. This also improves sorting (and grouping style) of imports in that file (which were being changed already to no longer import the old Commit_ish type). --- git/index/base.py | 43 +++++++++++++++---------------------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 668607772..80fa90ee8 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -11,23 +11,16 @@ import glob from io import BytesIO import os +import os.path as osp from stat import S_ISLNK import subprocess import sys import tempfile -from git.compat import ( - force_bytes, - defenc, -) -from git.exc import GitCommandError, CheckoutError, GitError, InvalidGitRepositoryError -from git.objects import ( - Blob, - Submodule, - Tree, - Object, - Commit, -) +from git.compat import defenc, force_bytes +import git.diff as git_diff +from git.exc import CheckoutError, GitCommandError, GitError, InvalidGitRepositoryError +from git.objects import Blob, Commit, Object, Submodule, Tree from git.objects.util import Serializable from git.util import ( LazyMixin, @@ -41,24 +34,17 @@ from gitdb.base import IStream from gitdb.db import MemoryDB -import git.diff as git_diff -import os.path as osp - from .fun import ( + S_IFGITLINK, + aggressive_tree_merge, entry_key, - write_cache, read_cache, - aggressive_tree_merge, - write_tree_from_cache, - stat_mode_to_index_mode, - S_IFGITLINK, run_commit_hook, + stat_mode_to_index_mode, + write_cache, + write_tree_from_cache, ) -from .typ import ( - BaseIndexEntry, - IndexEntry, - StageType, -) +from .typ import BaseIndexEntry, IndexEntry, StageType from .util import TemporaryFileSwap, post_clear_cache, default_index, git_working_dir # typing ----------------------------------------------------------------------------- @@ -80,12 +66,13 @@ Union, ) -from git.types import Old_commit_ish, Literal, PathLike +from git.types import Literal, PathLike if TYPE_CHECKING: from subprocess import Popen - from git.repo import Repo + from git.refs.reference import Reference + from git.repo import Repo from git.util import Actor @@ -1127,7 +1114,7 @@ def move( def commit( self, message: str, - parent_commits: Union[Old_commit_ish, None] = None, + parent_commits: Union[List[Commit], None] = None, head: bool = True, author: Union[None, "Actor"] = None, committer: Union[None, "Actor"] = None, From 5b2869faccbda659e12f2b1e7ca78b69ba770a11 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 Mar 2024 23:40:06 -0400 Subject: [PATCH 0811/1392] Fix old Commit_ish annotations in git.remote The combination of changes here is a bit unintuitive. - In PushInfo.__init__, the old_commit parameter is an optional string, whose value is bound to the _old_commit_sha instance attribute. The old_commit property attempts to create a Commit object using this string. When _old_commit_sha is None, this property returns None. Otherwise it calls Repo.commit on the Repo object for the PushInfo's associated remote. Repo.commit should return, and is annotated to return, a Commit object. Its return value is produced by calling rev_parse, which is actually the implementation in git.fun, which always returns an Object (and more specifically an AnyGitObject) and whose annotation was recently fixed in fe7f9f2. The way rev_parse is used appears to only be able to get a Commit, but even if it were to get something else, it would still be an Object and not a SymbolicReference or string. Therefore, the return annotation, which contained the old Commit_ish, was overly broad, in part because the old Commit_ish was defined over-broadly, but also in other ways. This changes it to declare that it returns a Commit object or None, not allowing any kind of refs, strings, or instances representing git objects other than commits. The new return annotation is also what type checkers infer for the operand of the return statement, ever since git.fun.rev_parse's own return annotation was fixed. - In contrast, the situation with FetchInfo is almost the opposite. FetchInfo.__init__ had declared its old_commit parameter as being of the old Commit_ish type or None. Given the name old_commit, this may seem at first glance to be a case where only actually commit-ish values should be accepted, such that the annotation may have been overly broad due the overbroad old definition of Commit_ish. But on closer inspection it seems that may not be so. Specifically, when "git fetch" reports the refs it updated, and a ref points to something that is not commit-ish, the "old_commit" can be something that is not a commit. In particular, if a remote lightweight tag points to a blob or tree (rather than the usual case of pointing to a commit or tag), and that tag is then changed, then a fetch -- if done with flags that allow it to be reset -- should result in an "old_commit" of the blob or tree. More specifically, in FetchInfo._from_line (in the "tag update" case), in a line with ".." or "...", old_commit is set by parsing a field with repo.rev_parse, which is git.fun.rev_parse, which will return an Object that may be any of the four types. This is later passed as the old_commmit parameter when constructing a FetchInfo instance. Since a lightweight tag is a ref that can refer to any of the four types, any could end up being parsed into the old_commit attribute. This therefore keeps those as broad as before, channging their old Commit_ish annotations to AnyGitObject rather than to the newer Commit_ish type or anything narrower. Note also that it is pure coincidence that entities named old_commit were temporarily annotated as Old_commit_ish. (The latter, as noted in 04a2753, is just what the old Commit_ish type was temporarily renamed to.) --- git/remote.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/git/remote.py b/git/remote.py index 01fe3771d..e0dc08639 100644 --- a/git/remote.py +++ b/git/remote.py @@ -41,7 +41,7 @@ overload, ) -from git.types import PathLike, Literal, Old_commit_ish +from git.types import AnyGitObject, Literal, PathLike if TYPE_CHECKING: from git.objects.commit import Commit @@ -194,7 +194,7 @@ def __init__( self.summary = summary @property - def old_commit(self) -> Union[str, SymbolicReference, Old_commit_ish, None]: + def old_commit(self) -> Union["Commit", None]: return self._old_commit_sha and self._remote.repo.commit(self._old_commit_sha) or None @property @@ -360,7 +360,7 @@ def __init__( ref: SymbolicReference, flags: int, note: str = "", - old_commit: Union[Old_commit_ish, None] = None, + old_commit: Union[AnyGitObject, None] = None, remote_ref_path: Optional[PathLike] = None, ) -> None: """Initialize a new instance.""" @@ -436,7 +436,7 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": # Parse operation string for more info. # This makes no sense for symbolic refs, but we parse it anyway. - old_commit: Union[Old_commit_ish, None] = None + old_commit: Union[AnyGitObject, None] = None is_tag_operation = False if "rejected" in operation: flags |= cls.REJECTED From dd8ee4fabb8ae7921cf53b95b195f4d2186f136f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Mar 2024 12:06:18 -0400 Subject: [PATCH 0812/1392] Start fixing venv test fixture pip toml bug This is not yet a usable fix, because venv.create only supports upgrade_deps on Python 3.9 and higher. --- test/lib/helper.py | 6 +++--- test/test_index.py | 2 +- test/test_installation.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index 26469ed5d..cf7b37b08 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -403,13 +403,13 @@ class VirtualEnvironment: __slots__ = ("_env_dir",) - def __init__(self, env_dir, *, with_pip): + def __init__(self, env_dir, *, need_pip): if os.name == "nt": self._env_dir = osp.realpath(env_dir) - venv.create(self.env_dir, symlinks=False, with_pip=with_pip) + venv.create(self.env_dir, symlinks=False, with_pip=need_pip, upgrade_deps=need_pip) else: self._env_dir = env_dir - venv.create(self.env_dir, symlinks=True, with_pip=with_pip) + venv.create(self.env_dir, symlinks=True, with_pip=need_pip, upgrade_deps=need_pip) @property def env_dir(self): diff --git a/test/test_index.py b/test/test_index.py index fa64b82a2..206023bfe 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1060,7 +1060,7 @@ def test_hook_uses_shell_not_from_cwd(self, rw_dir, case): # from a venv may not run when copied outside of it, and a global interpreter # won't run when copied to a different location if it was installed from the # Microsoft Store. So we make a new venv in rw_dir and use its interpreter. - venv = VirtualEnvironment(rw_dir, with_pip=False) + venv = VirtualEnvironment(rw_dir, need_pip=False) shutil.copy(venv.python, Path(rw_dir, shell_name)) shutil.copy(fixture_path("polyglot"), hook_path("polyglot", repo.git_dir)) payload = Path(rw_dir, "payload.txt") diff --git a/test/test_installation.py b/test/test_installation.py index ae6472e98..38c0c45b6 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -64,7 +64,7 @@ def test_installation(self, rw_dir): @staticmethod def _set_up_venv(rw_dir): - venv = VirtualEnvironment(rw_dir, with_pip=True) + venv = VirtualEnvironment(rw_dir, need_pip=True) os.symlink( os.path.dirname(os.path.dirname(__file__)), venv.sources, From a262a06634ccb616f061fdae7cf67b00a92f0ba6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Mar 2024 13:16:15 -0400 Subject: [PATCH 0813/1392] Upgrade test fixture pip in venv without upgrade_deps Because the upgrade_deps parameter to venv.create, as well as related functionality such as the EnvBuilder.upgrade_dependencies method that it uses, are only available starting in Python 3.9. This also puts the name of the VirtualEnvironment.__init__ parameter for setting up pip in the test fixture virtual environment back from need_pip to with_pip, which may be more intuitive. --- test/lib/helper.py | 15 ++++++++++++--- test/test_index.py | 2 +- test/test_installation.py | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index cf7b37b08..27586c2b0 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -10,6 +10,8 @@ import logging import os import os.path as osp +import subprocess +import sys import tempfile import textwrap import time @@ -403,13 +405,20 @@ class VirtualEnvironment: __slots__ = ("_env_dir",) - def __init__(self, env_dir, *, need_pip): + def __init__(self, env_dir, *, with_pip): if os.name == "nt": self._env_dir = osp.realpath(env_dir) - venv.create(self.env_dir, symlinks=False, with_pip=need_pip, upgrade_deps=need_pip) + venv.create(self.env_dir, symlinks=False, with_pip=with_pip) else: self._env_dir = env_dir - venv.create(self.env_dir, symlinks=True, with_pip=need_pip, upgrade_deps=need_pip) + venv.create(self.env_dir, symlinks=True, with_pip=with_pip) + + if with_pip: + # The upgrade_deps parameter to venv.create is 3.9+ only, so do it this way. + command = [self.python, "-m", "pip", "install", "--upgrade", "pip"] + if sys.version_info < (3, 12): + command.append("setuptools") + subprocess.check_output(command) @property def env_dir(self): diff --git a/test/test_index.py b/test/test_index.py index 206023bfe..fa64b82a2 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1060,7 +1060,7 @@ def test_hook_uses_shell_not_from_cwd(self, rw_dir, case): # from a venv may not run when copied outside of it, and a global interpreter # won't run when copied to a different location if it was installed from the # Microsoft Store. So we make a new venv in rw_dir and use its interpreter. - venv = VirtualEnvironment(rw_dir, need_pip=False) + venv = VirtualEnvironment(rw_dir, with_pip=False) shutil.copy(venv.python, Path(rw_dir, shell_name)) shutil.copy(fixture_path("polyglot"), hook_path("polyglot", repo.git_dir)) payload = Path(rw_dir, "payload.txt") diff --git a/test/test_installation.py b/test/test_installation.py index 38c0c45b6..ae6472e98 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -64,7 +64,7 @@ def test_installation(self, rw_dir): @staticmethod def _set_up_venv(rw_dir): - venv = VirtualEnvironment(rw_dir, need_pip=True) + venv = VirtualEnvironment(rw_dir, with_pip=True) os.symlink( os.path.dirname(os.path.dirname(__file__)), venv.sources, From 0b7f15945cc5adeb79ca9e25abafdbf8c565d716 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 11 Mar 2024 09:54:30 +0100 Subject: [PATCH 0814/1392] lint: replace `flake8` with `ruff` check --- .flake8 | 26 -------------------------- .pre-commit-config.yaml | 13 ++++++------- README.md | 2 +- pyproject.toml | 38 +++++++++++++++++++++++++++++++++++++- requirements-dev.txt | 2 -- 5 files changed, 44 insertions(+), 37 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 1cc049a69..000000000 --- a/.flake8 +++ /dev/null @@ -1,26 +0,0 @@ -[flake8] - -show-source = True -count = True -statistics = True - -# E266 = too many leading '#' for block comment -# E731 = do not assign a lambda expression, use a def -# TC002 = move third party import to TYPE_CHECKING -# TC, TC2 = flake8-type-checking - -# select = C,E,F,W ANN, TC, TC2 # to enable code. Disabled if not listed, including builtin codes -enable-extensions = TC, TC2 # only needed for extensions not enabled by default - -ignore = E266, E731 - -exclude = .tox, .venv, build, dist, doc, git/ext/ - -rst-roles = # for flake8-RST-docstrings - attr, class, func, meth, mod, obj, ref, term, var # used by sphinx - -min-python-version = 3.7.0 - -# for `black` compatibility -max-line-length = 120 -extend-ignore = E203, W503 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ac5baa00..cd5f58441 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,14 +14,13 @@ repos: name: black (format) exclude: ^git/ext/ -- repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.0 hooks: - - id: flake8 - additional_dependencies: - - flake8-bugbear==23.9.16 - - flake8-comprehensions==3.14.0 - - flake8-typing-imports==1.14.0 + #- id: ruff-format # todo: eventually replace Black with Ruff for consistency + # args: ["--preview"] + - id: ruff + args: ["--fix"] exclude: ^doc|^git/ext/ - repo: https://github.com/shellcheck-py/shellcheck-py diff --git a/README.md b/README.md index 7faeae23b..1e4a59d7f 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ The same linting, and running tests on all the different supported Python versio Specific tools: - Configurations for `mypy`, `pytest`, `coverage.py`, and `black` are in `./pyproject.toml`. -- Configuration for `flake8` is in the `./.flake8` file. +- Configuration for `ruff` is in the `pyproject.toml` file. Orchestration tools: diff --git a/pyproject.toml b/pyproject.toml index 7109389d7..230a9cf3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,6 @@ warn_unreachable = true show_error_codes = true implicit_reexport = true # strict = true - # TODO: Remove when 'gitdb' is fully annotated. exclude = ["^git/ext/gitdb"] [[tool.mypy.overrides]] @@ -47,3 +46,40 @@ omit = ["*/git/ext/*"] line-length = 120 target-version = ["py37"] extend-exclude = "git/ext/gitdb" + +[tool.ruff] +target-version = "py37" +line-length = 120 +# Exclude a variety of commonly ignored directories. +exclude = [ + "git/ext/", + "doc", + "build", + "dist", +] +# Enable Pyflakes `E` and `F` codes by default. +lint.select = [ + "E", + "W", # see: https://pypi.org/project/pycodestyle + "F", # see: https://pypi.org/project/pyflakes +# "I", #see: https://pypi.org/project/isort/ +# "S", # see: https://pypi.org/project/flake8-bandit +# "UP", # see: https://docs.astral.sh/ruff/rules/#pyupgrade-up +] +lint.extend-select = [ + "A", # see: https://pypi.org/project/flake8-builtins + "B", # see: https://pypi.org/project/flake8-bugbear + "C4", # see: https://pypi.org/project/flake8-comprehensions + "TCH004", # see: https://docs.astral.sh/ruff/rules/runtime-import-in-type-checking-block/ +] +lint.ignore = [ + "E203", "W503" +] +lint.ignore-init-module-imports = true +lint.unfixable = ["F401"] + +#[tool.ruff.lint.per-file-ignores] +#"setup.py" = ["ANN202", "ANN401"] +#"docs/source/conf.py" = ["A001", "D103"] +#"src/**" = ["ANN401"] +#"tests/**" = ["S101", "ANN001", "ANN201", "ANN202", "ANN401"] diff --git a/requirements-dev.txt b/requirements-dev.txt index e3030c597..69a79d13d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,5 @@ # libraries for additional local testing/linting - to be added to test-requirements.txt when all pass -flake8-type-checking;python_version>="3.8" # checks for TYPE_CHECKING only imports - pytest-icdiff # pytest-profiling From aa9298a37f1bbdecfbfe9b13026c81374abdc2d7 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 11 Mar 2024 11:10:00 +0100 Subject: [PATCH 0815/1392] fixing lints / noqa --- git/index/base.py | 8 ++++---- git/objects/submodule/base.py | 2 +- git/objects/util.py | 2 +- git/refs/symbolic.py | 4 ++-- pyproject.toml | 12 +++++------- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 249144d54..985b1bccf 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -248,7 +248,7 @@ def write( # Make sure we have our entries read before getting a write lock. # Otherwise it would be done when streaming. # This can happen if one doesn't change the index, but writes it right away. - self.entries + self.entries # noqa: B018 lfd = LockedFD(file_path or self._file_path) stream = lfd.open(write=True, stream=True) @@ -397,7 +397,7 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile with TemporaryFileSwap(join_path_native(repo.git_dir, "index")): repo.git.read_tree(*arg_list, **kwargs) index = cls(repo, tmp_index) - index.entries # Force it to read the file as we will delete the temp-file. + index.entries # noqa: B018 # Force it to read the file as we will delete the temp-file. return index # END index merge handling @@ -1339,7 +1339,7 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik # Make sure we have our entries loaded before we start checkout_index, which # will hold a lock on it. We try to get the lock as well during our entries # initialization. - self.entries + self.entries # noqa: B018 args.append("--stdin") kwargs["as_process"] = True @@ -1379,7 +1379,7 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik self._flush_stdin_and_wait(proc, ignore_stdout=True) except GitCommandError: # Without parsing stdout we don't know what failed. - raise CheckoutError( + raise CheckoutError( # noqa: B904 "Some files could not be checked out from the index, probably because they didn't exist.", failed_files, [], diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index cdd7c8e1b..b3e681e94 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1445,7 +1445,7 @@ def exists(self) -> bool: try: try: - self.path + self.path # noqa: B018 return True except Exception: return False diff --git a/git/objects/util.py b/git/objects/util.py index 6f4e7d087..71eb9c230 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -439,7 +439,7 @@ def _list_traverse( if not as_edge: out: IterableList[Union["Commit", "Submodule", "Tree", "Blob"]] = IterableList(id) - out.extend(self.traverse(as_edge=as_edge, *args, **kwargs)) + out.extend(self.traverse(as_edge=as_edge, *args, **kwargs)) # noqa: B026 return out # Overloads in subclasses (mypy doesn't allow typing self: subclass). # Union[IterableList['Commit'], IterableList['Submodule'], IterableList[Union['Submodule', 'Tree', 'Blob']]] diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 16aada0a7..465acf872 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -496,7 +496,7 @@ def is_valid(self) -> bool: valid object or reference. """ try: - self.object + self.object # noqa: B018 except (OSError, ValueError): return False else: @@ -510,7 +510,7 @@ def is_detached(self) -> bool: instead to another reference. """ try: - self.ref + self.ref # noqa: B018 return False except TypeError: return True diff --git a/pyproject.toml b/pyproject.toml index 230a9cf3a..af0e52ca4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,19 +67,17 @@ lint.select = [ # "UP", # see: https://docs.astral.sh/ruff/rules/#pyupgrade-up ] lint.extend-select = [ - "A", # see: https://pypi.org/project/flake8-builtins + #"A", # see: https://pypi.org/project/flake8-builtins "B", # see: https://pypi.org/project/flake8-bugbear "C4", # see: https://pypi.org/project/flake8-comprehensions "TCH004", # see: https://docs.astral.sh/ruff/rules/runtime-import-in-type-checking-block/ ] lint.ignore = [ - "E203", "W503" + "E203", + "E731", # Do not assign a `lambda` expression, use a `def` ] lint.ignore-init-module-imports = true lint.unfixable = ["F401"] -#[tool.ruff.lint.per-file-ignores] -#"setup.py" = ["ANN202", "ANN401"] -#"docs/source/conf.py" = ["A001", "D103"] -#"src/**" = ["ANN401"] -#"tests/**" = ["S101", "ANN001", "ANN201", "ANN202", "ANN401"] +[tool.ruff.lint.per-file-ignores] +"test/**" = ["B018"] From 5f889c4137d1d4442ab3ceb8e45c3c03cafded1a Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 11 Mar 2024 11:18:06 +0100 Subject: [PATCH 0816/1392] try: from typing import Literal --- git/objects/blob.py | 6 +++++- git/objects/commit.py | 7 ++++++- git/objects/submodule/base.py | 7 ++++++- git/objects/tag.py | 5 ++++- git/objects/tree.py | 7 ++++++- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/git/objects/blob.py b/git/objects/blob.py index 253ceccb5..4035c3e7c 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -6,7 +6,11 @@ from mimetypes import guess_type from . import base -from git.types import Literal + +try: + from typing import Literal +except ImportError: + from typing_extensions import Literal __all__ = ("Blob",) diff --git a/git/objects/commit.py b/git/objects/commit.py index 06ab0898b..dcb3be695 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -44,7 +44,12 @@ Dict, ) -from git.types import PathLike, Literal +from git.types import PathLike + +try: + from typing import Literal +except ImportError: + from typing_extensions import Literal if TYPE_CHECKING: from git.repo import Repo diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index b3e681e94..e5933b116 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -44,7 +44,12 @@ from typing import Callable, Dict, Mapping, Sequence, TYPE_CHECKING, cast from typing import Any, Iterator, Union -from git.types import Commit_ish, Literal, PathLike, TBD +from git.types import Commit_ish, PathLike, TBD + +try: + from typing import Literal +except ImportError: + from typing_extensions import Literal if TYPE_CHECKING: from git.index import IndexFile diff --git a/git/objects/tag.py b/git/objects/tag.py index f455c55fc..d8815e436 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -16,7 +16,10 @@ from typing import List, TYPE_CHECKING, Union -from git.types import Literal +try: + from typing import Literal +except ImportError: + from typing_extensions import Literal if TYPE_CHECKING: from git.repo import Repo diff --git a/git/objects/tree.py b/git/objects/tree.py index a506bba7d..731ab5fa1 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -31,7 +31,12 @@ TYPE_CHECKING, ) -from git.types import PathLike, Literal +from git.types import PathLike + +try: + from typing import Literal +except ImportError: + from typing_extensions import Literal if TYPE_CHECKING: from git.repo import Repo From 517f83aae949511fcc0696d4b67392d024acded1 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 11 Mar 2024 21:33:19 +0100 Subject: [PATCH 0817/1392] lint: switch Black with `ruff-format` --- .github/workflows/lint.yml | 2 -- .pre-commit-config.yaml | 18 ++---------------- Makefile | 2 +- pyproject.toml | 5 ----- test-requirements.txt | 1 - tox.ini | 2 -- 6 files changed, 3 insertions(+), 27 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d805124fc..a32fb6c4e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,5 +16,3 @@ jobs: - uses: pre-commit/action@v3.0.1 with: extra_args: --all-files --hook-stage manual - env: - SKIP: black-format diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cd5f58441..60bbe3518 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,24 +1,10 @@ repos: -- repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 - hooks: - - id: black - alias: black-check - name: black (check) - args: [--check, --diff] - exclude: ^git/ext/ - stages: [manual] - - - id: black - alias: black-format - name: black (format) - exclude: ^git/ext/ - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.3.0 hooks: - #- id: ruff-format # todo: eventually replace Black with Ruff for consistency - # args: ["--preview"] + - id: ruff-format + exclude: ^git/ext/ - id: ruff args: ["--fix"] exclude: ^doc|^git/ext/ diff --git a/Makefile b/Makefile index 839dc9f78..71a370ef2 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: @awk -F: '/^[[:alpha:]].*:/ && !/^all:/ {print $$1}' Makefile lint: - SKIP=black-format pre-commit run --all-files --hook-stage manual + SKIP=pre-commit run --all-files --hook-stage manual clean: rm -rf build/ dist/ .eggs/ .tox/ diff --git a/pyproject.toml b/pyproject.toml index af0e52ca4..eb57cc7b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,11 +42,6 @@ source = ["git"] include = ["*/git/*"] omit = ["*/git/ext/*"] -[tool.black] -line-length = 120 -target-version = ["py37"] -extend-exclude = "git/ext/gitdb" - [tool.ruff] target-version = "py37" line-length = 120 diff --git a/test-requirements.txt b/test-requirements.txt index 7cfb977a1..e1f5e2ed4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,3 @@ -black coverage[toml] ddt >= 1.1.1, != 1.4.3 mock ; python_version < "3.8" diff --git a/tox.ini b/tox.ini index f9ac25b78..28b7b147f 100644 --- a/tox.ini +++ b/tox.ini @@ -12,8 +12,6 @@ commands = pytest --color=yes {posargs} [testenv:lint] description = Lint via pre-commit base_python = py{39,310,311,312,38,37} -set_env = - SKIP = black-format commands = pre-commit run --all-files --hook-stage manual [testenv:mypy] From 2d0158c2c147dae90f9bbceeead1c4f322e90436 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 11 Mar 2024 21:36:25 +0100 Subject: [PATCH 0818/1392] apply `ruff-format` --- git/cmd.py | 72 ++++++++++++++++++--------------------------- git/compat.py | 18 ++++-------- git/index/fun.py | 6 ++-- git/objects/fun.py | 6 ++-- git/objects/tree.py | 2 +- git/objects/util.py | 12 +++----- git/remote.py | 11 +++---- git/util.py | 9 ++---- 8 files changed, 52 insertions(+), 84 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 731d0fab8..915f46a05 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -495,9 +495,8 @@ def refresh(cls, path: Union[None, PathLike] = None) -> bool: if mode in quiet: pass elif mode in warn or mode in error: - err = ( - dedent( - """\ + err = dedent( + """\ %s All git commands will error until this is rectified. @@ -510,16 +509,14 @@ def refresh(cls, path: Union[None, PathLike] = None) -> bool: Example: export %s=%s """ - ) - % ( - err, - cls._refresh_env_var, - "|".join(quiet), - "|".join(warn), - "|".join(error), - cls._refresh_env_var, - quiet[0], - ) + ) % ( + err, + cls._refresh_env_var, + "|".join(quiet), + "|".join(warn), + "|".join(error), + cls._refresh_env_var, + quiet[0], ) if mode in warn: @@ -527,9 +524,8 @@ def refresh(cls, path: Union[None, PathLike] = None) -> bool: else: raise ImportError(err) else: - err = ( - dedent( - """\ + err = dedent( + """\ %s environment variable has been set but it has been set with an invalid value. Use only the following values: @@ -537,13 +533,11 @@ def refresh(cls, path: Union[None, PathLike] = None) -> bool: - %s: for a warning message (logging level CRITICAL, displayed by default) - %s: for a raised exception """ - ) - % ( - cls._refresh_env_var, - "|".join(quiet), - "|".join(warn), - "|".join(error), - ) + ) % ( + cls._refresh_env_var, + "|".join(quiet), + "|".join(warn), + "|".join(error), ) raise ImportError(err) @@ -565,13 +559,11 @@ def is_cygwin(cls) -> bool: @overload @classmethod - def polish_url(cls, url: str, is_cygwin: Literal[False] = ...) -> str: - ... + def polish_url(cls, url: str, is_cygwin: Literal[False] = ...) -> str: ... @overload @classmethod - def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> str: - ... + def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> str: ... @classmethod def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> PathLike: @@ -932,8 +924,7 @@ def execute( command: Union[str, Sequence[Any]], *, as_process: Literal[True], - ) -> "AutoInterrupt": - ... + ) -> "AutoInterrupt": ... @overload def execute( @@ -942,8 +933,7 @@ def execute( *, as_process: Literal[False] = False, stdout_as_string: Literal[True], - ) -> Union[str, Tuple[int, str, str]]: - ... + ) -> Union[str, Tuple[int, str, str]]: ... @overload def execute( @@ -952,8 +942,7 @@ def execute( *, as_process: Literal[False] = False, stdout_as_string: Literal[False] = False, - ) -> Union[bytes, Tuple[int, bytes, str]]: - ... + ) -> Union[bytes, Tuple[int, bytes, str]]: ... @overload def execute( @@ -963,8 +952,7 @@ def execute( with_extended_output: Literal[False], as_process: Literal[False], stdout_as_string: Literal[True], - ) -> str: - ... + ) -> str: ... @overload def execute( @@ -974,8 +962,7 @@ def execute( with_extended_output: Literal[False], as_process: Literal[False], stdout_as_string: Literal[False], - ) -> bytes: - ... + ) -> bytes: ... def execute( self, @@ -1387,8 +1374,9 @@ def __call__(self, **kwargs: Any) -> "Git": return self @overload - def _call_process(self, method: str, *args: None, **kwargs: None) -> str: - ... # If no args were given, execute the call with all defaults. + def _call_process( + self, method: str, *args: None, **kwargs: None + ) -> str: ... # If no args were given, execute the call with all defaults. @overload def _call_process( @@ -1398,14 +1386,12 @@ def _call_process( as_process: Literal[True], *args: Any, **kwargs: Any, - ) -> "Git.AutoInterrupt": - ... + ) -> "Git.AutoInterrupt": ... @overload def _call_process( self, method: str, *args: Any, **kwargs: Any - ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], "Git.AutoInterrupt"]: - ... + ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], "Git.AutoInterrupt"]: ... def _call_process( self, method: str, *args: Any, **kwargs: Any diff --git a/git/compat.py b/git/compat.py index 7753fe8b2..e64c645c7 100644 --- a/git/compat.py +++ b/git/compat.py @@ -69,13 +69,11 @@ @overload -def safe_decode(s: None) -> None: - ... +def safe_decode(s: None) -> None: ... @overload -def safe_decode(s: AnyStr) -> str: - ... +def safe_decode(s: AnyStr) -> str: ... def safe_decode(s: Union[AnyStr, None]) -> Optional[str]: @@ -91,13 +89,11 @@ def safe_decode(s: Union[AnyStr, None]) -> Optional[str]: @overload -def safe_encode(s: None) -> None: - ... +def safe_encode(s: None) -> None: ... @overload -def safe_encode(s: AnyStr) -> bytes: - ... +def safe_encode(s: AnyStr) -> bytes: ... def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]: @@ -113,13 +109,11 @@ def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]: @overload -def win_encode(s: None) -> None: - ... +def win_encode(s: None) -> None: ... @overload -def win_encode(s: AnyStr) -> bytes: - ... +def win_encode(s: AnyStr) -> bytes: ... def win_encode(s: Optional[AnyStr]) -> Optional[bytes]: diff --git a/git/index/fun.py b/git/index/fun.py index 58335739e..beca67d3f 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -286,9 +286,9 @@ def read_cache( # 4 bytes length of chunk # Repeated 0 - N times extension_data = stream.read(~0) - assert ( - len(extension_data) > 19 - ), "Index Footer was not at least a sha on content as it was only %i bytes in size" % len(extension_data) + assert len(extension_data) > 19, ( + "Index Footer was not at least a sha on content as it was only %i bytes in size" % len(extension_data) + ) content_sha = extension_data[-20:] diff --git a/git/objects/fun.py b/git/objects/fun.py index 22b99cb6b..5bd8a3d62 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -152,13 +152,11 @@ def _find_by_name(tree_data: MutableSequence[EntryTupOrNone], name: str, is_dir: @overload -def _to_full_path(item: None, path_prefix: str) -> None: - ... +def _to_full_path(item: None, path_prefix: str) -> None: ... @overload -def _to_full_path(item: EntryTup, path_prefix: str) -> EntryTup: - ... +def _to_full_path(item: EntryTup, path_prefix: str) -> EntryTup: ... def _to_full_path(item: EntryTupOrNone, path_prefix: str) -> EntryTupOrNone: diff --git a/git/objects/tree.py b/git/objects/tree.py index 731ab5fa1..3964b016c 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -188,7 +188,7 @@ class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): _map_id_to_type: Dict[int, Type[IndexObjUnion]] = { commit_id: Submodule, blob_id: Blob, - symlink_id: Blob + symlink_id: Blob, # Tree ID added once Tree is defined. } diff --git a/git/objects/util.py b/git/objects/util.py index 71eb9c230..26a34f94c 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -620,8 +620,7 @@ def list_traverse(self: T_TIobj, *args: Any, **kwargs: Any) -> IterableList[T_TI return super()._list_traverse(*args, **kwargs) @overload # type: ignore - def traverse(self: T_TIobj) -> Iterator[T_TIobj]: - ... + def traverse(self: T_TIobj) -> Iterator[T_TIobj]: ... @overload def traverse( @@ -633,8 +632,7 @@ def traverse( visit_once: bool, ignore_self: Literal[True], as_edge: Literal[False], - ) -> Iterator[T_TIobj]: - ... + ) -> Iterator[T_TIobj]: ... @overload def traverse( @@ -646,8 +644,7 @@ def traverse( visit_once: bool, ignore_self: Literal[False], as_edge: Literal[True], - ) -> Iterator[Tuple[Union[T_TIobj, None], T_TIobj]]: - ... + ) -> Iterator[Tuple[Union[T_TIobj, None], T_TIobj]]: ... @overload def traverse( @@ -659,8 +656,7 @@ def traverse( visit_once: bool, ignore_self: Literal[True], as_edge: Literal[True], - ) -> Iterator[Tuple[T_TIobj, T_TIobj]]: - ... + ) -> Iterator[Tuple[T_TIobj, T_TIobj]]: ... def traverse( self: T_TIobj, diff --git a/git/remote.py b/git/remote.py index fd4de7100..b63cfc208 100644 --- a/git/remote.py +++ b/git/remote.py @@ -93,22 +93,19 @@ def add_progress( @overload -def to_progress_instance(progress: None) -> RemoteProgress: - ... +def to_progress_instance(progress: None) -> RemoteProgress: ... @overload -def to_progress_instance(progress: Callable[..., Any]) -> CallableRemoteProgress: - ... +def to_progress_instance(progress: Callable[..., Any]) -> CallableRemoteProgress: ... @overload -def to_progress_instance(progress: RemoteProgress) -> RemoteProgress: - ... +def to_progress_instance(progress: RemoteProgress) -> RemoteProgress: ... def to_progress_instance( - progress: Union[Callable[..., Any], RemoteProgress, None] + progress: Union[Callable[..., Any], RemoteProgress, None], ) -> Union[RemoteProgress, CallableRemoteProgress]: """Given the `progress` return a suitable object derived from :class:`~git.util.RemoteProgress`.""" diff --git a/git/util.py b/git/util.py index 9d451eee2..27751f687 100644 --- a/git/util.py +++ b/git/util.py @@ -441,13 +441,11 @@ def decygpath(path: PathLike) -> str: @overload -def is_cygwin_git(git_executable: None) -> Literal[False]: - ... +def is_cygwin_git(git_executable: None) -> Literal[False]: ... @overload -def is_cygwin_git(git_executable: PathLike) -> bool: - ... +def is_cygwin_git(git_executable: PathLike) -> bool: ... def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: @@ -494,8 +492,7 @@ def finalize_process(proc: Union[subprocess.Popen, "Git.AutoInterrupt"], **kwarg @overload -def expand_path(p: None, expand_vars: bool = ...) -> None: - ... +def expand_path(p: None, expand_vars: bool = ...) -> None: ... @overload From 1b8812a968a76f47acdf80bc1d15e6b0dd5d84bf Mon Sep 17 00:00:00 2001 From: Jirka Date: Tue, 12 Mar 2024 21:08:54 +0100 Subject: [PATCH 0819/1392] drop `make lint` --- Makefile | 5 +---- README.md | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 71a370ef2..d4f9acf87 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,8 @@ -.PHONY: all lint clean release force_release +.PHONY: all clean release force_release all: @awk -F: '/^[[:alpha:]].*:/ && !/^all:/ {print $$1}' Makefile -lint: - SKIP=pre-commit run --all-files --hook-stage manual - clean: rm -rf build/ dist/ .eggs/ .tox/ diff --git a/README.md b/README.md index 1e4a59d7f..33e093945 100644 --- a/README.md +++ b/README.md @@ -165,9 +165,6 @@ To lint, and apply automatic code formatting, run: pre-commit run --all-files ``` -- Linting without modifying code can be done with: `make lint` -- Auto-formatting without other lint checks can be done with: `black .` - To typecheck, run: ```bash From 395b70ae2fdf01590f157e94d96e7ddac7893ba9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Mar 2024 12:12:27 -0400 Subject: [PATCH 0820/1392] Very slightly improve readme presentation - Add a missing period. - Indicate sh rather than bash as as the language for syntax highlighting of shell commands that don't need bash. I had held off on making that second change in previous revisions because it would have involved either introducing an inconsistency or editing the section giving the deprecated signature-checking instructions. That section was removed in 2671167 (#1823). (This also wraps a paragraph where the immediately surrounding text was wrapped, but that should not affect rendered text, and broader consistency improvements to Markdown wrapping style are not done.) --- README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 33e093945..458162212 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ probably the skills to scratch that itch of mine: implement `git` in a way that If you like the idea and want to learn more, please head over to [gitoxide](https://github.com/Byron/gitoxide), an implementation of 'git' in [Rust](https://www.rust-lang.org). -*(Please note that `gitoxide` is not currently available for use in Python, and that Rust is required)* +*(Please note that `gitoxide` is not currently available for use in Python, and that Rust is required.)* ## GitPython @@ -39,9 +39,9 @@ The project is open to contributions of all kinds, as well as new maintainers. ### REQUIREMENTS -GitPython needs the `git` executable to be installed on the system and available in your `PATH` for most operations. -If it is not in your `PATH`, you can help GitPython find it by setting -the `GIT_PYTHON_GIT_EXECUTABLE=` environment variable. +GitPython needs the `git` executable to be installed on the system and available in your +`PATH` for most operations. If it is not in your `PATH`, you can help GitPython find it +by setting the `GIT_PYTHON_GIT_EXECUTABLE=` environment variable. - Git (1.7.x or newer) - Python >= 3.7 @@ -57,7 +57,7 @@ GitPython and its required package dependencies can be installed in any of the f To obtain and install a copy [from PyPI](https://pypi.org/project/GitPython/), run: -```bash +```sh pip install GitPython ``` @@ -67,7 +67,7 @@ pip install GitPython If you have downloaded the source code, run this from inside the unpacked `GitPython` directory: -```bash +```sh pip install . ``` @@ -75,7 +75,7 @@ pip install . To clone the [the GitHub repository](https://github.com/gitpython-developers/GitPython) from source to work on the code, you can do it like so: -```bash +```sh git clone https://github.com/gitpython-developers/GitPython cd GitPython ./init-tests-after-clone.sh @@ -85,7 +85,7 @@ On Windows, `./init-tests-after-clone.sh` can be run in a Git Bash shell. If you are cloning [your own fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks), then replace the above `git clone` command with one that gives the URL of your fork. Or use this [`gh`](https://cli.github.com/) command (assuming you have `gh` and your fork is called `GitPython`): -```bash +```sh gh repo clone GitPython ``` @@ -93,7 +93,7 @@ Having cloned the repo, create and activate your [virtual environment](https://d Then make an [editable install](https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs): -```bash +```sh pip install -e ".[test]" ``` @@ -105,7 +105,7 @@ In rare cases, you may want to work on GitPython and one or both of its [gitdb]( If you want to do that *and* you want the versions in GitPython's git submodules to be used, then pass `-e git/ext/gitdb` and/or `-e git/ext/gitdb/gitdb/ext/smmap` to `pip install`. This can be done in any order, and in separate `pip install` commands or the same one, so long as `-e` appears before *each* path. For example, you can install GitPython, gitdb, and smmap editably in the currently active virtual environment this way: -```bash +```sh pip install -e ".[test]" -e git/ext/gitdb -e git/ext/gitdb/gitdb/ext/smmap ``` @@ -141,13 +141,13 @@ you will encounter test failures. Ensure testing libraries are installed. This is taken care of already if you installed with: -```bash +```sh pip install -e ".[test]" ``` Otherwise, you can run: -```bash +```sh pip install -r test-requirements.txt ``` @@ -155,19 +155,19 @@ pip install -r test-requirements.txt To test, run: -```bash +```sh pytest ``` To lint, and apply automatic code formatting, run: -```bash +```sh pre-commit run --all-files ``` To typecheck, run: -```bash +```sh mypy -p git ``` From 3a6ee9e0e478eea5f6defe2b851a7ca6c74976ca Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Mar 2024 12:31:06 -0400 Subject: [PATCH 0821/1392] Make installation instructions more consistent If people who want to run the tests didn't install the test extra, they can still install that extra. This simplifies the instructions accordingly. test-requirements.txt is still mentioned near the beginning in case people want to look at it to see dependencies. But the changed code is the only place where the instructions had still said to do anything with those files. A possible disadvantage of this change is that in the rare case that someone following those instructions to run the tests locally didn't do an editable installation, then installing with the extra shouldn't be done editably either. But this doesn't seem like a likely problem, and the new text has an example command with -e to clarify the kind of command whose effect is augmented here. Because of that subtlety, it is not obvious that this change is really justified for purposes of clarity. However, this also helps prepare for a time when test-requirements.txt won't exist anymore, as may happen when the project definition is made fully declarative (see discussion in comments in #1716). --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 458162212..ead8093f8 100644 --- a/README.md +++ b/README.md @@ -145,11 +145,8 @@ Ensure testing libraries are installed. This is taken care of already if you ins pip install -e ".[test]" ``` -Otherwise, you can run: - -```sh -pip install -r test-requirements.txt -``` +If you had installed with a command like `pip install -e .` instead, you can still run +the above command to add the testing dependencies. #### Test commands From 826234384d38126130388102355a274a353bfade Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Mar 2024 12:50:41 -0400 Subject: [PATCH 0822/1392] Update readme for recent tooling changes This also reorganizes the "Specific tools" list, since they are all configured in pyproject.toml now (only flake8 was not before, and it was removed in favor of ruff in #1862). In doing so, I've also added brief parenthesized phrases to characterize what each of these four tools is for, so readers don't have to look around as much to understand most of the tooling GitPython has set up. --- README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ead8093f8..30af532db 100644 --- a/README.md +++ b/README.md @@ -156,12 +156,14 @@ To test, run: pytest ``` -To lint, and apply automatic code formatting, run: +To lint, and apply some linting fixes as well as automatic code formatting, run: ```sh pre-commit run --all-files ``` +This includes the linting and autoformatting done by Ruff, as well as some other checks. + To typecheck, run: ```sh @@ -170,7 +172,7 @@ mypy -p git #### CI (and tox) -The same linting, and running tests on all the different supported Python versions, will be performed: +Style and formatting checks, and running tests on all the different supported Python versions, will be performed: - Upon submitting a pull request. - On each push, *if* you have a fork with GitHub Actions enabled. @@ -178,10 +180,12 @@ The same linting, and running tests on all the different supported Python versio #### Configuration files -Specific tools: +Specific tools are all configured in the `./pyproject.toml` file: -- Configurations for `mypy`, `pytest`, `coverage.py`, and `black` are in `./pyproject.toml`. -- Configuration for `ruff` is in the `pyproject.toml` file. +- `pytest` (test runner) +- `coverage.py` (code coverage) +- `ruff` (linter and formatter) +- `mypy` (type checker) Orchestration tools: From b059cd580b71b44488bf3cc77000a6dea3cb1898 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 12 Mar 2024 13:27:34 -0400 Subject: [PATCH 0823/1392] Have tox skip linting unless requested, for now This is to make it so simple `tox` usage has the expected property of leaving all source code files in the working tree unchanged. Linting how sometimes performs auto-fixes since #1862, and the pre-commit command in tox.ini, which had also run `black --check`, will do even more file editing due to the changes in #1865. The bifurcation for black into separate mutating and non-mutating hooks, introduced in 5d8ddd9 (#1693), was not carried over into Ruff autoformatting in #1865. But also it: - Was not necessarily a good approach, and likely should not be preserved in any form. It was an unusual and unintuitive use of pre-commit. (It can be brought back if no better approach is found, though.) - Was done to avoid a situation where it was nontrivial to set up necessary dependencies for linting in the GitPython virtual environment itself, because flake8 and its various plugins would have to be installed. They were not listed in any existing or newly introduced extra (for example, they were not added to test-requirements.txt) in part in the hope that they would all be replaced by Ruff, which happened in #1862. - Already did not achieve its goal as of #1862, since it was (probably rightly) not extended to Ruff linting to use/omit --fix. Now that Ruff is being used, people can run `pip install ruff` in a virtual environment, then run the `ruff` command however they like. This takes the place of multiple tools and plugins. The situation with the tox "lint" environment is thus now similar to that of the tox "html" environment when it was added in e6ec6c8 (#1667), until it was improved in f094909 (#1693) to run with proper isolation. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 28b7b147f..6e02e5aee 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = tox>=4 -env_list = py{37,38,39,310,311,312}, lint, mypy, html +env_list = py{37,38,39,310,311,312}, mypy, html [testenv] description = Run unit tests From f78841870cea95926a95ccaae000bd51769ba412 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Mar 2024 10:46:53 -0400 Subject: [PATCH 0824/1392] Clean up mention of manual hook stage This is no longer used. No pre-commit hook specifies it anymore in `stages`, since 517f83a (#1865). See b059cd5 (#1868) for context. In the lint.yml GitHub Actions workflow, this removes the extra_args key altogether, because all that would remain there is --all-files, which is already the default for that action, when the extra_args key is absent. --- .github/workflows/lint.yml | 2 -- tox.ini | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a32fb6c4e..a0e81a993 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,5 +14,3 @@ jobs: python-version: "3.x" - uses: pre-commit/action@v3.0.1 - with: - extra_args: --all-files --hook-stage manual diff --git a/tox.ini b/tox.ini index 6e02e5aee..31ac9382d 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ commands = pytest --color=yes {posargs} [testenv:lint] description = Lint via pre-commit base_python = py{39,310,311,312,38,37} -commands = pre-commit run --all-files --hook-stage manual +commands = pre-commit run --all-files [testenv:mypy] description = Typecheck with mypy From 4c034db60e05f73104d265ec53099a8b00522269 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Mar 2024 11:27:07 -0400 Subject: [PATCH 0825/1392] Split tox "lint" env into three envs, all safe It makes sense for ruff linting and ruff autoformatting to be easily runnable individually and to have their results be shown separately. Splitting them out in tox.ini also makes it so tox can do the other tests corresponding to those in lint.yml on CI in an environment that requries no custom behavior from pre-commit other than skipping the ruff checks. The ruff linting ("ruff") and ruff format checking ("format") tox environments specify ruff as a dependency and call it directly rather than through pre-commit, invoking it in such a way that it does not attempt to modify any files in the working tree. See b059cd5 (#1868) for context. All three of these tox envs that "lint" has been split into are listed in the env_list and thus run automatically when tox is run with no arguments, since no tox envs unexpectedly (or at all) modify files in the working tree anymore. One limitation of the current approach is that new pre-commit hooks configured in .pre-commit-config.yml will automatically be run as part of the "misc" tox environment, which means that new unexpected mutating operatons could be added if the impact on tox is not considered. The benefit of having it work this way is that most hooks that are likely to be added to GitPython would not modify files and would be wanted as part of "misc". But this may benefit from further refinement. --- tox.ini | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 31ac9382d..1fc7e2415 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = tox>=4 -env_list = py{37,38,39,310,311,312}, mypy, html +env_list = py{37,38,39,310,311,312}, ruff, format, mypy, html, misc [testenv] description = Run unit tests @@ -9,10 +9,17 @@ extras = test pass_env = SSH_* commands = pytest --color=yes {posargs} -[testenv:lint] -description = Lint via pre-commit +[testenv:ruff] +description = Lint with Ruff base_python = py{39,310,311,312,38,37} -commands = pre-commit run --all-files +deps = ruff +commands = ruff check . + +[testenv:format] +description = Check formatting with Ruff +base_python = py{39,310,311,312,38,37} +deps = ruff +commands = ruff format --check . [testenv:mypy] description = Typecheck with mypy @@ -28,3 +35,10 @@ allowlist_externals = make commands = make BUILDDIR={env_tmp_dir}/doc/build -C doc clean make BUILDDIR={env_tmp_dir}/doc/build -C doc html + +[testenv:misc] +description = Run other checks via pre-commit +base_python = py{39,310,311,312,38,37} +set_env = + SKIP = ruff-format,ruff +commands = pre-commit run --all-files From dcbd5dbc38d95dfd31dd5897cb7511a7fda0cce3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Mar 2024 11:38:44 -0400 Subject: [PATCH 0826/1392] Colorize ruff output when run through tox In most other situations it is typically already colorized. This also includes comments about how to override this to suppress color. --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index 1fc7e2415..48c9ed2f9 100644 --- a/tox.ini +++ b/tox.ini @@ -13,12 +13,16 @@ commands = pytest --color=yes {posargs} description = Lint with Ruff base_python = py{39,310,311,312,38,37} deps = ruff +set_env = + CLICOLOR_FORCE = 1 # Set NO_COLOR to override this. commands = ruff check . [testenv:format] description = Check formatting with Ruff base_python = py{39,310,311,312,38,37} deps = ruff +set_env = + CLICOLOR_FORCE = 1 # Set NO_COLOR to override this. commands = ruff format --check . [testenv:mypy] From 89e519e05ca3342c3747b19901b8bb6c1d2b7f58 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Mar 2024 11:47:50 -0400 Subject: [PATCH 0827/1392] Colorize mypy output when run through tox On Unix-like systems, this approach is currently only sufficient if the TERM environment variable is set. In practice this means that if tox is run on CI, color will not be shown. Because GitPython does not itself use tox on CI, and other (e.g. downstream) projects may not want color on CI or in other situations where this would be insufficient, the further step of defining TERM as a workaround is deliberately omitted. See aeacb0 in #1859 for what adding TERM would look like. This does not make any changes to how mypy runs on CI because that change is already included there. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 48c9ed2f9..d6c23bf60 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,8 @@ commands = ruff format --check . [testenv:mypy] description = Typecheck with mypy base_python = py{39,310,311,312,38,37} +set_env = + MYPY_FORCE_COLOR = 1 commands = mypy -p git ignore_outcome = true From 00ff7e33c4c44adb9d75f92a1590cb3e3dc45729 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Mar 2024 13:17:39 -0400 Subject: [PATCH 0828/1392] Let Ruff scan doc/source/conf.py And other .py files in doc/, if there ever were any. This may have been turned off before either for performance or out of a concern than some flake8 plugin would catch something in the docs themselves, but that's no longer an issue. The doc/build directory, which should still not be scanned, is already excluded in other ways (and does not need to be excluded for pre-commit to improve performance, since it is untracked). --- .pre-commit-config.yaml | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 60bbe3518..f269678e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: exclude: ^git/ext/ - id: ruff args: ["--fix"] - exclude: ^doc|^git/ext/ + exclude: ^git/ext/ - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.9.0.5 diff --git a/pyproject.toml b/pyproject.toml index eb57cc7b7..e8f3d720e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,6 @@ line-length = 120 # Exclude a variety of commonly ignored directories. exclude = [ "git/ext/", - "doc", "build", "dist", ] From 4814775bfee418d7e0cc9ffb0ac30a4e44a48d75 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Mar 2024 13:32:28 -0400 Subject: [PATCH 0829/1392] Comment all Ruff rule codes; tweak formatting This adds comments to all specific Ruff rules suppressed in pyproject.toml (most of which are global but one is for the tests only). Some already had such comments but now all do. This also harmonizes the toml formatting style with the rest of pyproject.toml. --- pyproject.toml | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e8f3d720e..57ae01058 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,24 +54,28 @@ exclude = [ # Enable Pyflakes `E` and `F` codes by default. lint.select = [ "E", - "W", # see: https://pypi.org/project/pycodestyle - "F", # see: https://pypi.org/project/pyflakes -# "I", #see: https://pypi.org/project/isort/ -# "S", # see: https://pypi.org/project/flake8-bandit -# "UP", # see: https://docs.astral.sh/ruff/rules/#pyupgrade-up + "W", # See: https://pypi.org/project/pycodestyle + "F", # See: https://pypi.org/project/pyflakes + # "I", # See: https://pypi.org/project/isort/ + # "S", # See: https://pypi.org/project/flake8-bandit + # "UP", # See: https://docs.astral.sh/ruff/rules/#pyupgrade-up ] lint.extend-select = [ - #"A", # see: https://pypi.org/project/flake8-builtins - "B", # see: https://pypi.org/project/flake8-bugbear - "C4", # see: https://pypi.org/project/flake8-comprehensions - "TCH004", # see: https://docs.astral.sh/ruff/rules/runtime-import-in-type-checking-block/ + # "A", # See: https://pypi.org/project/flake8-builtins + "B", # See: https://pypi.org/project/flake8-bugbear + "C4", # See: https://pypi.org/project/flake8-comprehensions + "TCH004", # See: https://docs.astral.sh/ruff/rules/runtime-import-in-type-checking-block/ ] lint.ignore = [ - "E203", - "E731", # Do not assign a `lambda` expression, use a `def` + "E203", # Whitespace before ':' + "E731", # Do not assign a `lambda` expression, use a `def` ] lint.ignore-init-module-imports = true -lint.unfixable = ["F401"] +lint.unfixable = [ + "F401", # Module imported but unused +] [tool.ruff.lint.per-file-ignores] -"test/**" = ["B018"] +"test/**" = [ + "B018", # useless-expression +] From a8a73ff739785a01a528b2a274c8252456a1ebc2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Mar 2024 13:44:57 -0400 Subject: [PATCH 0830/1392] Update requirements-dev.txt (as long as we have it) Although it seems likely that the requirements-dev.txt file will be removed when the project definition is made declarative (discussed in #1716 comments), if not before, for now it exists and might be in use, so this updates it with tools that are currently used but not listed in any extras or other requirements files: - ruff: This has replaced flake8 and its plugins (#1862) as well as black (#1865). Currently there is no separate extra for tooling that is not part of unit testing, with some such tools listed in test-requirements.txt. The `ruff` package belongs here rather than there for now because it should not be installed just to run unit tests, since Ruff has to be built from source on some rarer platforms like Cygwin, which would take a long time and/or fail. - shellcheck: The PyPI package for this is a convenience for installing it in projects that are already using pip (shellcheck is neither written in Python nor a tool to scan Python code). It installs pre-built binaries, which are not available for all platforms. These packages remain listed: - pytest-icdiff: This seems not to have been promoted to be in the test-requirements.txt file and the `test` extra because it does not work as well for diffs from tests that the pytest runner runs but that are written for the unittest framework. - pytest-profiling [commented out]: I am not sure what the status of this is, perhaps it has just not been experimented with enough to know if it would be useful for profiling in GitPython. This requirements-dev.txt file has a few limitations that suggest it should be removed altogether sometime soon: - It is not updated regularly. - It is not always clear why something is there. Originally I believe it was for tools where the desire to use the tool was established but the tool did not yet work or worked but performed checks for which code had to be fixed. That purpose has drifted. - It uses a different naming convention from the test-requirements.txt file in active use. - It cannot be readily used to create an extra in the current project definition in setup.py because the simple parsing done there will not recognize the `-r` lines and will not skip the comments, and neither enhancement should be done in setup.py since that would move things farther away from a declarative project definition. - It will naturally go away when the project definition is made declarative, since it will then be feasible to define as many extras as desired without proliferating separate requirements files (while still allowing their contents to be statically available to tools). Since it may go away soon and is not regularly updated, I have kept the explanations for why particular packages are there out of it. But as long as it exists it may as well list the tools that really are being used yet are not explicitly listed as dependencies. --- requirements-dev.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 69a79d13d..f626644af 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,8 @@ -r requirements.txt -r test-requirements.txt -# libraries for additional local testing/linting - to be added to test-requirements.txt when all pass - +# For additional local testing/linting - to be added elsewhere eventually. +ruff +shellcheck pytest-icdiff # pytest-profiling From ff1ebf8fe05f2b5bace15dc7edbc6bdc4aea2222 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Mar 2024 14:11:16 -0400 Subject: [PATCH 0831/1392] Bump pre-commit hook versions The changes, per hook repository, are: - ruff-pre-commit: Uses the newest stable version of Ruff. See: * https://github.com/astral-sh/ruff-pre-commit/releases/tag/v0.3.2 - shellcheck-py: Makes ShellCheck installable on ARM64 Windows machines. See: * https://github.com/shellcheck-py/shellcheck-py/compare/v0.9.0.5...v0.9.0.6 * https://github.com/shellcheck-py/shellcheck-py/commit/e39ba22d34cf877af3e6529a253a6cfd3e9f8ea7 - pre-commit-hooks: Offers no relevant changes for the few hooks currently being used from that repository, but I went ahead and included this while bumping the others. This may be a small benefit if more hooks are added from it sometime soon and because we don't currently have a mechanism for keeping track of updates that are skipped as unneeded. --- .pre-commit-config.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f269678e4..585b4f04d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,6 @@ repos: - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.0 + rev: v0.3.2 hooks: - id: ruff-format exclude: ^git/ext/ @@ -10,14 +9,14 @@ repos: exclude: ^git/ext/ - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.5 + rev: v0.9.0.6 hooks: - id: shellcheck args: [--color] exclude: ^test/fixtures/polyglot$|^git/ext/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-toml - id: check-yaml From 1b43166951caac29ed223c9d1522534626b3598e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Mar 2024 17:09:03 -0400 Subject: [PATCH 0832/1392] Group setup.py imports While setuptools has official status, it is not actually part of the standard library (and since Python 3.12 cannot be treated as if it is, since it is not installed by default), so its imports belong in the second group rather than the first, per PEP-8. --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 73d1ae952..e827d2483 100755 --- a/setup.py +++ b/setup.py @@ -1,11 +1,13 @@ #!/usr/bin/env python +import os +import sys from typing import Sequence + from setuptools import setup, find_packages from setuptools.command.build_py import build_py as _build_py from setuptools.command.sdist import sdist as _sdist -import os -import sys + with open(os.path.join(os.path.dirname(__file__), "VERSION"), encoding="utf-8") as ver_file: VERSION = ver_file.readline().strip() From 26dccd70713b2be6cb0af6892fd05703f17cda3e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Mar 2024 17:49:56 -0400 Subject: [PATCH 0833/1392] Streamline setup.py file reading This is partly a refactoring, reducing logic duplication and slightly decreasing overall length of the code that reads values from files. However, it does make two small behavioral changes: - In addition to the VERSION file for which this was already the case, requirements files and the readme are also taken relative to the directory that contains the setup.py file, rather than relative to the current directory if different. Supporting reading the readme and dependencies realtive to a different directory doesn't seem intended, and could lead to confusion in some cases. - The VERSION file is now read entirely and whitespace stripped, rather than reading just the first line. The VERSION file should always contain exactly one line, and this logic is simpler. This changes code for the reading of metadata only. It does not make any changes to the version stamping code. --- setup.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index e827d2483..3f57e3327 100755 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ import os import sys +from pathlib import Path from typing import Sequence from setuptools import setup, find_packages @@ -9,17 +10,14 @@ from setuptools.command.sdist import sdist as _sdist -with open(os.path.join(os.path.dirname(__file__), "VERSION"), encoding="utf-8") as ver_file: - VERSION = ver_file.readline().strip() +def _read_content(path: str) -> str: + return (Path(__file__).parent / path).read_text(encoding="utf-8") -with open("requirements.txt", encoding="utf-8") as reqs_file: - requirements = reqs_file.read().splitlines() -with open("test-requirements.txt", encoding="utf-8") as reqs_file: - test_requirements = reqs_file.read().splitlines() - -with open("README.md", encoding="utf-8") as rm_file: - long_description = rm_file.read() +version = _read_content("VERSION").strip() +requirements = _read_content("requirements.txt").splitlines() +test_requirements = _read_content("test-requirements.txt").splitlines() +long_description = _read_content("README.md") class build_py(_build_py): @@ -50,7 +48,7 @@ def _stamp_version(filename: str) -> None: with open(filename) as f: for line in f: if "__version__ =" in line: - line = line.replace('"git"', "'%s'" % VERSION) + line = line.replace('"git"', "'%s'" % version) found = True out.append(line) except OSError: @@ -66,7 +64,7 @@ def _stamp_version(filename: str) -> None: setup( name="GitPython", cmdclass={"build_py": build_py, "sdist": sdist}, - version=VERSION, + version=version, description="GitPython is a Python library used to interact with Git repositories", author="Sebastian Thiel, Michael Trier", author_email="byronimo@gmail.com, mtrier@gmail.com", From 74df5a8995b6f4e9ed053e126dda1cb6cfc465f5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Mar 2024 18:29:50 -0400 Subject: [PATCH 0834/1392] Add a "doc" extra for documentation build dependencies And use it in: - GitHub Actions CI checks - Read the Docs configuration - tox (for the "html" environment) (The tox "html" environment was not previously overriding "extras" to empty it out of dependencies needed only for linting or testing, so this happens to make `tox -e html` faster.) --- .github/workflows/pythonpackage.yml | 2 +- .readthedocs.yaml | 3 ++- setup.py | 6 +++++- tox.ini | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 6b89530c3..4ef741c36 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -100,5 +100,5 @@ jobs: - name: Documentation run: | - pip install -r doc/requirements.txt + pip install ".[doc]" make -C doc html diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 0b83e20ea..9bce80fd2 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -32,4 +32,5 @@ python: install: - method: pip path: . - - requirements: doc/requirements.txt + extra_requirements: + - doc diff --git a/setup.py b/setup.py index 3f57e3327..143206653 100755 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ def _read_content(path: str) -> str: version = _read_content("VERSION").strip() requirements = _read_content("requirements.txt").splitlines() test_requirements = _read_content("test-requirements.txt").splitlines() +doc_requirements = _read_content("doc/requirements.txt").splitlines() long_description = _read_content("README.md") @@ -75,7 +76,10 @@ def _stamp_version(filename: str) -> None: package_dir={"git": "git"}, python_requires=">=3.7", install_requires=requirements, - extras_require={"test": test_requirements}, + extras_require={ + "test": test_requirements, + "doc": doc_requirements, + }, zip_safe=False, long_description=long_description, long_description_content_type="text/markdown", diff --git a/tox.ini b/tox.ini index 6e02e5aee..dfcb5ed8f 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ ignore_outcome = true [testenv:html] description = Build HTML documentation base_python = py{39,310,311,312,38,37} -deps = -r doc/requirements.txt +extras = doc allowlist_externals = make commands = make BUILDDIR={env_tmp_dir}/doc/build -C doc clean From 1541c62d86e334980d008d1e8ff0c6623ae2e2a0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 11 Mar 2024 01:06:17 -0400 Subject: [PATCH 0835/1392] Start on fixing Submodule parent_commit annotations These used the old Commit_ish. They may not all be the same as each other, since they perform different validation and some will look up a commit while others do not. In particular, this is complicated by how Submodile.__init__ documents its parent_commit parameter: it says to see the set_parent_commit method. But __init__ sets the _parent_commit attribute to parent_commit, while set_parent_commit treats its commit parameter differently, calling self.repo.commit on it to get get a Commit object from it even if it wasn't one to begin with. See #1869 for full details. There may be some other subtleties as well. A number of uses of the of the old Commit_ish type appear in git.submodule.base, all related to parent_commit. These are all remaining references to it (identifiable due to the rename to Old_commit_ish done in 04a2753), though. So once the changes begun here for git.submodule.* are done, that will also have completed the broader replacement effort begun in 7328a00. At that point cleanup can be done in git.types (removing Old_commit_ish introduced in 04a27531, replacing Lit_old_commit_ish with a possibly-updated and deprecated Lit_commit_ish to avoid a breaking change, and possibly making further refinements to additions to the newly added docstrings). --- git/objects/submodule/base.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 783990664..2a8c62b70 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -45,12 +45,13 @@ from typing import Callable, Dict, Mapping, Sequence, TYPE_CHECKING, cast from typing import Any, Iterator, Union -from git.types import Old_commit_ish, Literal, PathLike, TBD +from git.types import Commit_ish, Literal, Old_commit_ish, PathLike, TBD if TYPE_CHECKING: from git.index import IndexFile - from git.repo import Repo + from git.objects.commit import Commit from git.refs import Head + from git.repo import Repo # ----------------------------------------------------------------------------- @@ -99,7 +100,7 @@ class Submodule(IndexObject, TraversableIterableObj): """Submodule flags. Submodules are directories with link-status.""" type: Literal["submodule"] = "submodule" # type: ignore - """This is a bogus type for base class compatibility.""" + """This is a bogus type string for base class compatibility.""" __slots__ = ("_parent_commit", "_url", "_branch_path", "_name", "__weakref__") @@ -1242,7 +1243,7 @@ def remove( return self - def set_parent_commit(self, commit: Union[Old_commit_ish, None], check: bool = True) -> "Submodule": + def set_parent_commit(self, commit: Union[Commit_ish, str, None], check: bool = True) -> "Submodule": """Set this instance to use the given commit whose tree is supposed to contain the ``.gitmodules`` blob. @@ -1495,7 +1496,7 @@ def url(self) -> str: return self._url @property - def parent_commit(self) -> "Old_commit_ish": + def parent_commit(self) -> "Commit": """ :return: :class:`~git.objects.commit.Commit` instance with the tree containing the @@ -1557,8 +1558,8 @@ def children(self) -> IterableList["Submodule"]: def iter_items( cls, repo: "Repo", - parent_commit: Union[Old_commit_ish, str] = "HEAD", - *Args: Any, + parent_commit: Union[Commit_ish, str] = "HEAD", + *args: Any, **kwargs: Any, ) -> Iterator["Submodule"]: """ From 1f03e7fc23745c64de7a54a92c79ddfa96cd1025 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 13 Mar 2024 23:15:08 -0400 Subject: [PATCH 0836/1392] Fix other submodule.base parent_commit annotations This finishes the work in git.objects.submodule.base begun in 1541c62. Some remaining occurrences of the old Commit_ish type are in git.objects.submodule.root, which still need to be replaced. Even once that is done, this will not have fixed #1869, though it will be somewhat mitigated because each method's annotations related to parent_commit will reflect the types that can actually be used reliably under the current implementation, even if in some cases that may not be as broad as was meant, or hoped, to be supported. I expect this may lead the way to a fix for #1869, since the existing behavior and relationships will be clearer. Where the old Commit_ish annotations were changed here to "Commit", it reflects assumptions found by examining the implementation that other alternatives are not usable becuse something beloning to a Commit is used and no conversion or fallback is performed for it. Where a commit is looked up (which uses rev_parse), broader types are annotated. Where the argument is used without conversion and must have a `tree` attribute, it must be a Commit. Where the argument could be broader but must be compared meaningfully with something known to be a commit, it must at minimum have a binsha attribute to allow that equality comparison, implemented in Object, to be meaningful, so in such cases its type cannot include str. --- git/objects/submodule/base.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 2a8c62b70..61757092a 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -42,10 +42,19 @@ # typing ---------------------------------------------------------------------- -from typing import Callable, Dict, Mapping, Sequence, TYPE_CHECKING, cast -from typing import Any, Iterator, Union +from typing import ( + Any, + Callable, + Dict, + Iterator, + Mapping, + Sequence, + TYPE_CHECKING, + Union, + cast, +) -from git.types import Commit_ish, Literal, Old_commit_ish, PathLike, TBD +from git.types import Commit_ish, Literal, PathLike, TBD if TYPE_CHECKING: from git.index import IndexFile @@ -113,7 +122,7 @@ def __init__( mode: Union[int, None] = None, path: Union[PathLike, None] = None, name: Union[str, None] = None, - parent_commit: Union[Old_commit_ish, None] = None, + parent_commit: Union["Commit", None] = None, url: Union[str, None] = None, branch_path: Union[PathLike, None] = None, ) -> None: @@ -145,7 +154,6 @@ def __init__( if url is not None: self._url = url if branch_path is not None: - # assert isinstance(branch_path, str) self._branch_path = branch_path if name is not None: self._name = name @@ -214,7 +222,7 @@ def __repr__(self) -> str: @classmethod def _config_parser( - cls, repo: "Repo", parent_commit: Union[Old_commit_ish, None], read_only: bool + cls, repo: "Repo", parent_commit: Union["Commit", None], read_only: bool ) -> SubmoduleConfigParser: """ :return: @@ -265,7 +273,7 @@ def _clear_cache(self) -> None: # END for each name to delete @classmethod - def _sio_modules(cls, parent_commit: Old_commit_ish) -> BytesIO: + def _sio_modules(cls, parent_commit: "Commit") -> BytesIO: """ :return: Configuration file as :class:`~io.BytesIO` - we only access it through the @@ -278,7 +286,7 @@ def _sio_modules(cls, parent_commit: Old_commit_ish) -> BytesIO: def _config_parser_constrained(self, read_only: bool) -> SectionConstraint: """:return: Config parser constrained to our submodule in read or write mode""" try: - pc: Union["Old_commit_ish", None] = self.parent_commit + pc = self.parent_commit except ValueError: pc = None # END handle empty parent repository From e66b8f1598343077e91fbb1c5483e22cdcdf0faa Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Mar 2024 00:07:47 -0400 Subject: [PATCH 0837/1392] Fix old Commit_ish annotation in RootModule This finishes the annotation fixup begun in 7328a00. (The temporary Old_commit_ish type is now ready to be removed.) --- git/objects/submodule/root.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index 8e697ee1d..f6a8c6807 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -1,18 +1,18 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import logging + +import git +from git.exc import InvalidGitRepositoryError from .base import Submodule, UpdateProgress from .util import find_first_remote_branch -from git.exc import InvalidGitRepositoryError -import git - -import logging # typing ------------------------------------------------------------------- from typing import TYPE_CHECKING, Union -from git.types import Old_commit_ish +from git.types import Commit_ish if TYPE_CHECKING: from git.repo import Repo @@ -77,7 +77,7 @@ def _clear_cache(self) -> None: def update( # type: ignore[override] self, - previous_commit: Union[Old_commit_ish, None] = None, + previous_commit: Union[Commit_ish, str, None] = None, recursive: bool = True, force_remove: bool = False, init: bool = True, From 93d19dcbc9a44607420f3d47c602419efc7b4b82 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Mar 2024 00:25:46 -0400 Subject: [PATCH 0838/1392] Remove the temporary Old_commit_ish type --- git/types.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/git/types.py b/git/types.py index 95a5816f3..d883d7324 100644 --- a/git/types.py +++ b/git/types.py @@ -126,31 +126,6 @@ gitglossary(7) on "object type": https://git-scm.com/docs/gitglossary#def_object_type """ -# FIXME: Replace uses with AnyGitObject and Commit_ish, and remove this. -Old_commit_ish = Union["Commit", "TagObject", "Blob", "Tree"] -"""Union of the :class:`~git.objects.base.Object`-based types that represent git object -types. This union is often usable where a commit-ish is expected, but is not actually -limited to types representing commit-ish git objects. - -See gitglossary(7) on: - -* "commit-ish": https://git-scm.com/docs/gitglossary#def_commit-ish -* "object type": https://git-scm.com/docs/gitglossary#def_object_type - -:note: - This union comprises **more** classes than those whose instances really represent - commit-ish git objects: - - * A :class:`~git.objects.commit.Commit` is of course always commit-ish, and a - :class:`~git.objects.tag.TagObject` is commit-ish if, when peeled (recursively - followed), a :class:`~git.objects.commit.Commit` is obtained. - * However, :class:`~git.objects.blob.Blob` and :class:`~git.objects.tree.Tree` are - also included, and they represent git objects that are never really commit-ish. - - This is an inversion of the situation with :class:`Tree_ish`, which is narrower than - all tree-ish objects. It is done for practical reasons including backward - compatibility. -""" # FIXME: After replacing the one use with GitObjectTypeString, define Lit_commit_ish # somehow (it is a breaking change to remove it entirely). Maybe deprecate it. From ebcfced0eebe0008ac5c11dad8b1fea7091424bd Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Mar 2024 01:11:05 -0400 Subject: [PATCH 0839/1392] Fix and deprecate Lit_commit_ish It would be a breaking change to keep Lit_commit_ish undefined, so this brings it back, but with a narrowed definition for consistency with the corrected Commit_ish, and deprecated, with instructions on what to use instead. (This also improves import sorting in the same module.) --- git/types.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/git/types.py b/git/types.py index d883d7324..336f49082 100644 --- a/git/types.py +++ b/git/types.py @@ -4,38 +4,38 @@ import os import sys from typing import ( # noqa: F401 + Any, + Callable, Dict, NoReturn, + Optional, Sequence as Sequence, Tuple, - Union, - Any, - Optional, - Callable, TYPE_CHECKING, TypeVar, + Union, ) if sys.version_info >= (3, 8): from typing import ( # noqa: F401 Literal, - TypedDict, Protocol, SupportsIndex as SupportsIndex, + TypedDict, runtime_checkable, ) else: from typing_extensions import ( # noqa: F401 Literal, + Protocol, SupportsIndex as SupportsIndex, TypedDict, - Protocol, runtime_checkable, ) if TYPE_CHECKING: - from git.repo import Repo from git.objects import Commit, Tree, TagObject, Blob + from git.repo import Repo PathLike = Union[str, "os.PathLike[str]"] """A :class:`str` (Unicode) based file or directory path.""" @@ -126,22 +126,19 @@ gitglossary(7) on "object type": https://git-scm.com/docs/gitglossary#def_object_type """ +Lit_commit_ish = Literal["commit", "tag"] +"""Deprecated. Type of literal strings identifying sometimes-commitish git object types. -# FIXME: After replacing the one use with GitObjectTypeString, define Lit_commit_ish -# somehow (it is a breaking change to remove it entirely). Maybe deprecate it. -Lit_old_commit_ish = Literal["commit", "tag", "blob", "tree"] -"""Literal strings identifying concrete :class:`~git.objects.base.Object` subtypes -representing git object types. - -See the :class:`Object.type ` attribute. +Prior to a bugfix, this type had been defined more broadly. Any usage is in practice +ambiguous and likely to be incorrect. Instead of this type: -:note: - See also :class:`Old_commit_ish`, a union of the the :class:`~git.objects.base.Object` - subtypes associated with these literal strings. +* For the type of the string literals associated with :class:`Commit_ish`, use + ``Literal["commit", "tag"]`` or create a new type alias for it. That is equivalent to + this type as currently defined. -:note: - As noted in :class:`Old_commit_ish`, this is not limited to types of git objects that - are actually commit-ish. +* For the type of all four string literals associated with :class:`AnyGitObject`, use + :class:`GitObjectTypeString`. That is equivalent to the old definition of this type + prior to the bugfix. """ # Config_levels --------------------------------------------------------- From b070e933fef5d1e86d3fb43c1613c03a760bbc50 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Mar 2024 01:37:27 -0400 Subject: [PATCH 0840/1392] Make some broad mypy suppressions more specific - Change every bare `# type: ignore` into `# type: ignore[reason]`, where the specific reason is verified by mypy. - Use separate `# type: ignore[X]` and `# type: ignore[Y]` for different parts of statement where a single bare suppression was doing double duty, with each suppression only on the specific parts (splitting the statement into multiple lines for this). - Use the same suppression twice on very long statements where only specific parts need suppressions. This also formats all these comments with the same spacing (most but not all already were). This makes them easier to grep for. --- git/__init__.py | 2 +- git/cmd.py | 10 +++++----- git/objects/commit.py | 8 ++++++-- git/objects/submodule/base.py | 2 +- git/objects/tree.py | 2 +- git/objects/util.py | 10 +++++++++- git/refs/remote.py | 2 +- git/refs/symbolic.py | 19 ++++++++++++++++--- git/repo/base.py | 2 +- git/util.py | 4 ++-- 10 files changed, 43 insertions(+), 18 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index 026dc1461..ca5bed7a3 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -147,7 +147,7 @@ def refresh(path: Optional[PathLike] = None) -> None: if not Git.refresh(path=path): return if not FetchInfo.refresh(): # noqa: F405 - return # type: ignore [unreachable] + return # type: ignore[unreachable] GIT_OK = True diff --git a/git/cmd.py b/git/cmd.py index 85398497f..74693b27a 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -172,7 +172,7 @@ def pump_stream( p_stdout = process.proc.stdout if process.proc else None p_stderr = process.proc.stderr if process.proc else None else: - process = cast(Popen, process) # type: ignore [redundant-cast] + process = cast(Popen, process) # type: ignore[redundant-cast] cmdline = getattr(process, "args", "") p_stdout = process.stdout p_stderr = process.stderr @@ -215,7 +215,7 @@ def pump_stream( error_str = error_str.encode() # We ignore typing on the next line because mypy does not like the way # we inferred that stderr takes str or bytes. - stderr_handler(error_str) # type: ignore + stderr_handler(error_str) # type: ignore[arg-type] if finalizer: finalizer(process) @@ -1244,9 +1244,9 @@ def communicate() -> Tuple[AnyStr, AnyStr]: if output_stream is None: stdout_value, stderr_value = communicate() # Strip trailing "\n". - if stdout_value.endswith(newline) and strip_newline_in_stdout: # type: ignore + if stdout_value.endswith(newline) and strip_newline_in_stdout: # type: ignore[arg-type] stdout_value = stdout_value[:-1] - if stderr_value.endswith(newline): # type: ignore + if stderr_value.endswith(newline): # type: ignore[arg-type] stderr_value = stderr_value[:-1] status = proc.returncode @@ -1256,7 +1256,7 @@ def communicate() -> Tuple[AnyStr, AnyStr]: stdout_value = proc.stdout.read() stderr_value = proc.stderr.read() # Strip trailing "\n". - if stderr_value.endswith(newline): # type: ignore + if stderr_value.endswith(newline): # type: ignore[arg-type] stderr_value = stderr_value[:-1] status = proc.wait() # END stdout handling diff --git a/git/objects/commit.py b/git/objects/commit.py index 6d3864be6..31aaf60f4 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -428,7 +428,11 @@ def trailers_list(self) -> List[Tuple[str, str]]: List containing key-value tuples of whitespace stripped trailer information. """ cmd = ["git", "interpret-trailers", "--parse"] - proc: Git.AutoInterrupt = self.repo.git.execute(cmd, as_process=True, istream=PIPE) # type: ignore + proc: Git.AutoInterrupt = self.repo.git.execute( # type: ignore[call-overload] + cmd, + as_process=True, + istream=PIPE, + ) trailer: str = proc.communicate(str(self.message).encode())[0].decode("utf8") trailer = trailer.strip() @@ -507,7 +511,7 @@ def _iter_from_process_or_stream(cls, repo: "Repo", proc_or_stream: Union[Popen, if proc_or_stream.stdout is not None: stream = proc_or_stream.stdout elif hasattr(proc_or_stream, "readline"): - proc_or_stream = cast(IO, proc_or_stream) # type: ignore [redundant-cast] + proc_or_stream = cast(IO, proc_or_stream) # type: ignore[redundant-cast] stream = proc_or_stream readline = stream.readline diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 61757092a..f9e0a8e0f 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -108,7 +108,7 @@ class Submodule(IndexObject, TraversableIterableObj): k_default_mode = stat.S_IFDIR | stat.S_IFLNK """Submodule flags. Submodules are directories with link-status.""" - type: Literal["submodule"] = "submodule" # type: ignore + type: Literal["submodule"] = "submodule" # type: ignore[assignment] """This is a bogus type string for base class compatibility.""" __slots__ = ("_parent_commit", "_url", "_branch_path", "_name", "__weakref__") diff --git a/git/objects/tree.py b/git/objects/tree.py index 30b6a9e4e..e9c08a7cd 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -391,7 +391,7 @@ def __contains__(self, item: Union[IndexObjUnion, PathLike]) -> bool: return False def __reversed__(self) -> Iterator[IndexObjUnion]: - return reversed(self._iter_convert_to_object(self._cache)) # type: ignore + return reversed(self._iter_convert_to_object(self._cache)) # type: ignore[call-overload] def _serialize(self, stream: "BytesIO") -> "Tree": """Serialize this tree into the stream. Assumes sorted tree data. diff --git a/git/objects/util.py b/git/objects/util.py index 194403ba5..576aca406 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -692,5 +692,13 @@ def traverse( return cast( Union[Iterator[T_TIobj], Iterator[Tuple[Union[None, T_TIobj], T_TIobj]]], - super()._traverse(predicate, prune, depth, branch_first, visit_once, ignore_self, as_edge), # type: ignore + super()._traverse( + predicate, # type: ignore[arg-type] + prune, # type: ignore[arg-type] + depth, + branch_first, + visit_once, + ignore_self, + as_edge, + ), ) diff --git a/git/refs/remote.py b/git/refs/remote.py index bb2a4e438..5cbd1b81b 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -52,7 +52,7 @@ def iter_items( # subclasses and recommends Any or "type: ignore". # (See: https://github.com/python/typing/issues/241) @classmethod - def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: # type: ignore + def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: # type: ignore[override] """Delete the given remote references. :note: diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 4a39875a7..1f1daa11b 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -387,8 +387,17 @@ def set_object( # set the commit on our reference return self._get_reference().set_object(object, logmsg) - commit = property(_get_commit, set_commit, doc="Query or set commits directly") # type: ignore - object = property(_get_object, set_object, doc="Return the object our ref currently refers to") # type: ignore + commit = property( + _get_commit, + set_commit, # type: ignore[arg-type] + doc="Query or set commits directly", + ) + + object = property( + _get_object, + set_object, # type: ignore[arg-type] + doc="Return the object our ref currently refers to", + ) def _get_reference(self) -> "SymbolicReference": """ @@ -488,7 +497,11 @@ def set_reference( # Aliased reference reference: Union["Head", "TagReference", "RemoteReference", "Reference"] - reference = property(_get_reference, set_reference, doc="Returns the Reference we point to") # type: ignore + reference = property( # type: ignore[assignment] + _get_reference, + set_reference, # type: ignore[arg-type] + doc="Returns the Reference we point to", + ) ref = reference def is_valid(self) -> bool: diff --git a/git/repo/base.py b/git/repo/base.py index 5e7645366..fe01a9279 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -636,7 +636,7 @@ def _get_config_path(self, config_level: Lit_config_levels, git_dir: Optional[Pa else: return osp.normpath(osp.join(repo_dir, "config")) else: - assert_never( # type:ignore[unreachable] + assert_never( # type: ignore[unreachable] config_level, ValueError(f"Invalid configuration level: {config_level!r}"), ) diff --git a/git/util.py b/git/util.py index 7ebce0c2c..8b60b1cb2 100644 --- a/git/util.py +++ b/git/util.py @@ -517,7 +517,7 @@ def expand_path(p: Union[None, PathLike], expand_vars: bool = True) -> Optional[ if isinstance(p, pathlib.Path): return p.resolve() try: - p = osp.expanduser(p) # type: ignore + p = osp.expanduser(p) # type: ignore[arg-type] if expand_vars: p = osp.expandvars(p) return osp.normpath(osp.abspath(p)) @@ -1209,7 +1209,7 @@ def __getattr__(self, attr: str) -> T_IterableObj: # END for each item return list.__getattribute__(self, attr) - def __getitem__(self, index: Union[SupportsIndex, int, slice, str]) -> T_IterableObj: # type: ignore + def __getitem__(self, index: Union[SupportsIndex, int, slice, str]) -> T_IterableObj: # type: ignore[override] assert isinstance(index, (int, str, slice)), "Index of IterableList should be an int or str" if isinstance(index, int): From 011cb0a134396e5f2713897e0c22f5124b156cf4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Mar 2024 02:35:34 -0400 Subject: [PATCH 0841/1392] Apply Ruff auto-fixes not included in merge --- git/index/base.py | 4 +--- git/repo/fun.py | 6 ++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 883d086f5..74ad65394 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1467,9 +1467,7 @@ def reset( # does not handle NULL_TREE for `other`. (The suppressed mypy error is about this.) def diff( self, - other: Union[ # type: ignore[override] - Literal[git_diff.DiffConstants.INDEX], "Tree", "Commit", str, None - ] = git_diff.INDEX, + other: Union[Literal[git_diff.DiffConstants.INDEX], "Tree", "Commit", str, None] = git_diff.INDEX, # type: ignore[override] paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, **kwargs: Any, diff --git a/git/repo/fun.py b/git/repo/fun.py index b65432b09..0ac481206 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -138,13 +138,11 @@ def short_to_long(odb: "GitCmdObjectDB", hexsha: str) -> Optional[bytes]: @overload -def name_to_object(repo: "Repo", name: str, return_ref: Literal[False] = ...) -> AnyGitObject: - ... +def name_to_object(repo: "Repo", name: str, return_ref: Literal[False] = ...) -> AnyGitObject: ... @overload -def name_to_object(repo: "Repo", name: str, return_ref: Literal[True]) -> Union[AnyGitObject, SymbolicReference]: - ... +def name_to_object(repo: "Repo", name: str, return_ref: Literal[True]) -> Union[AnyGitObject, SymbolicReference]: ... def name_to_object(repo: "Repo", name: str, return_ref: bool = False) -> Union[AnyGitObject, SymbolicReference]: From 74f3c2e7ad65e112d7e04644bc05b838c9c60d7f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Mar 2024 02:36:30 -0400 Subject: [PATCH 0842/1392] Help Ruff avoid a very long line --- git/index/base.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/git/index/base.py b/git/index/base.py index 74ad65394..59b019f0f 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1467,7 +1467,13 @@ def reset( # does not handle NULL_TREE for `other`. (The suppressed mypy error is about this.) def diff( self, - other: Union[Literal[git_diff.DiffConstants.INDEX], "Tree", "Commit", str, None] = git_diff.INDEX, # type: ignore[override] + other: Union[ # type: ignore[override] + Literal[git_diff.DiffConstants.INDEX], + "Tree", + "Commit", + str, + None, + ] = git_diff.INDEX, paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, **kwargs: Any, From 5778b7a01b988e711216fa5541cfdc50c0460476 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Mar 2024 02:37:50 -0400 Subject: [PATCH 0843/1392] Use LBYL for imports where EAFP is a mypy type error The conditional imports of Literal from either typing or typing_extensions have to be done with if-else on the Python version rather than with try-except, or it is a static type error. This makes that change, checking sys.version_info. (See discussion in #1861 and #1862 for broader context and background on why this logic, before and after this change, is repeated across multiple modules.) This also reorders/regroups imports for consistency in some places, especially where a new import of the sys module (for version_info) would otherwise exacerbate inconsistency. Since the merge commit 0b99041, the number of mypy errors had increased from 5 to 10. This fixes all the new mypy errors, so the count is back to 5. --- git/objects/blob.py | 7 ++++--- git/objects/commit.py | 38 +++++++++++++++++------------------ git/objects/submodule/base.py | 9 ++++----- git/objects/tag.py | 6 ++++-- git/objects/tree.py | 25 +++++++++++------------ 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/git/objects/blob.py b/git/objects/blob.py index 0a8527407..b49930edf 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -4,12 +4,13 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from mimetypes import guess_type -from . import base +import sys +from . import base -try: +if sys.version_info >= (3, 8): from typing import Literal -except ImportError: +else: from typing_extensions import Literal __all__ = ("Blob",) diff --git a/git/objects/commit.py b/git/objects/commit.py index f17e3fdf5..473eae8cc 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -3,57 +3,57 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +from collections import defaultdict import datetime +from io import BytesIO +import logging +import os import re from subprocess import Popen, PIPE +import sys +from time import altzone, daylight, localtime, time, timezone + from gitdb import IStream -from git.util import hex_to_bin, Actor, Stats, finalize_process -from git.diff import Diffable from git.cmd import Git +from git.diff import Diffable +from git.util import hex_to_bin, Actor, Stats, finalize_process from .tree import Tree -from . import base from .util import ( Serializable, TraversableIterableObj, - parse_date, altz_to_utctz_str, - parse_actor_and_date, from_timestamp, + parse_actor_and_date, + parse_date, ) - -from time import time, daylight, altzone, timezone, localtime -import os -from io import BytesIO -import logging -from collections import defaultdict - +from . import base # typing ------------------------------------------------------------------ from typing import ( Any, + Dict, IO, Iterator, List, Sequence, Tuple, - Union, TYPE_CHECKING, + Union, cast, - Dict, ) -from git.types import PathLike - -try: +if sys.version_info >= (3, 8): from typing import Literal -except ImportError: +else: from typing_extensions import Literal +from git.types import PathLike + if TYPE_CHECKING: - from git.repo import Repo from git.refs import SymbolicReference + from git.repo import Repo # ------------------------------------------------------------------------ diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 8c4a7356e..4e5a2a964 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -39,7 +39,6 @@ sm_section, ) - # typing ---------------------------------------------------------------------- from typing import ( @@ -54,13 +53,13 @@ cast, ) -from git.types import Commit_ish, PathLike, TBD - -try: +if sys.version_info >= (3, 8): from typing import Literal -except ImportError: +else: from typing_extensions import Literal +from git.types import Commit_ish, PathLike, TBD + if TYPE_CHECKING: from git.index import IndexFile from git.objects.commit import Commit diff --git a/git/objects/tag.py b/git/objects/tag.py index 83cf4ae18..e7ecfa62b 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -9,6 +9,8 @@ For lightweight tags, see the :mod:`git.refs.tag` module. """ +import sys + from . import base from .util import get_object_type_by_name, parse_actor_and_date from ..util import hex_to_bin @@ -16,9 +18,9 @@ from typing import List, TYPE_CHECKING, Union -try: +if sys.version_info >= (3, 8): from typing import Literal -except ImportError: +else: from typing_extensions import Literal if TYPE_CHECKING: diff --git a/git/objects/tree.py b/git/objects/tree.py index 225061bb7..308dd47a0 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -3,17 +3,16 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from git.util import IterableList, join_path +import sys + import git.diff as git_diff -from git.util import to_bin_sha +from git.util import IterableList, join_path, to_bin_sha -from . import util -from .base import IndexObject, IndexObjUnion +from .base import IndexObjUnion, IndexObject from .blob import Blob -from .submodule.base import Submodule - from .fun import tree_entries_from_data, tree_to_stream - +from .submodule.base import Submodule +from . import util # typing ------------------------------------------------- @@ -25,22 +24,22 @@ Iterator, List, Tuple, + TYPE_CHECKING, Type, Union, cast, - TYPE_CHECKING, ) -from git.types import PathLike - -try: +if sys.version_info >= (3, 8): from typing import Literal -except ImportError: +else: from typing_extensions import Literal +from git.types import PathLike + if TYPE_CHECKING: - from git.repo import Repo from io import BytesIO + from git.repo import Repo TreeCacheTup = Tuple[bytes, int, str] From 84e256d999f748f08bbd62bfa833f96664febcee Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Mar 2024 18:40:08 -0400 Subject: [PATCH 0844/1392] Describe Submodule.__init__ parent_commit parameter This includes a brief description of the Submodule.__init__ parent_commit parameter in its docstring, rather than only referring to the set_parent_commit method, whose semantics differ due to conversation and validation, and which accepts more types than just Commit or None. The wording is based on wording in set_parent_commit, adjusted for the difference in types, and set_parent_commit remains reference for further details. This builds on 1f03e7f (#1859) in improving the situation described in #1869. --- git/objects/submodule/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 4e5a2a964..0e51ae711 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -143,7 +143,9 @@ def __init__( See the `url` parameter. :param parent_commit: - See :meth:`set_parent_commit`. + The :class:`~git.objects.commit.Commit` whose tree is supposed to contain + the ``.gitmodules`` blob, or ``None`` to always point to the most recent + commit. See :meth:`set_parent_commit` for details. :param url: The URL to the remote repository which is the submodule. @@ -1260,7 +1262,7 @@ def set_parent_commit(self, commit: Union[Commit_ish, str, None], check: bool = contain the ``.gitmodules`` blob. :param commit: - Commit-ish reference pointing at the root_tree, or ``None`` to always point + Commit-ish reference pointing at the root tree, or ``None`` to always point to the most recent commit. :param check: From 70ef69ae118ef8f393eb3f5c84e72e24df09516b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 14 Mar 2024 21:36:49 -0400 Subject: [PATCH 0845/1392] Include TagObject in git.types.Tree_ish The Tree_ish union omitted TagObject, whose instances are only sometimes tree-ish, and unlike Commit_ish before #1859, it is not inherently a bug to define Tree_ish this way. However, this Tree_ish type actually has only one use in GitPython (which was also the case before the changes in #1859): as, itself, an alternative in the union used to annotate the rev parameter of the Repo.tree method (whose other alternatives are str and None). A TagObject may be passed, and if it points to a tree or commit then that will be resolved. Just to avoid a mypy error, code doing that would (before this change) have to convert it to str first. That annotation should be improved, and the best way to do it is to keep it written the same way but change the definition of Tree_ish in git.types to include TagObject. The reason is that doing so alleviates a major unintuitive aspect of the relationship between the Commit_ish and Tree_ish types: Commit_ish was broader than everything commit-ish, while Tree_ish was narrower than everything tree-ish. I had not considered making this change in #1859 because I didn't want to modify Tree_ish unnecessarily, and its definition was not inherently a bug. However, the change to Commit_ish is sufficiently large (though it only affects static typing) that a change to Tree_ish to make them coherent and intuitive may be justified. This commit changes Tree_ish so that, in addition to its Commit and Tree alternatives, it also includes TagObject. This also updates and simplifies its docstring accordingly, bringing it in line with that of Commit_ish which is already defined with the same kind of breadth, and further revises both docstrings to more explicitly clarify when tags are tree-ish or commit-ish and when they are not. This does not change the separate nonpublic Treeish type defined in git.index.base (and named with no underscore), which omits TagObject but includes bytes and str, and which is used to annotate parameters of the IndexFile.from_tree and IndexFile.merge_tree methods. Changes there may be valuable, but the goal here is just to build narrowly on #1859 to address a shortcoming of the revisions to git.types. --- git/types.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/git/types.py b/git/types.py index 336f49082..64f7629dc 100644 --- a/git/types.py +++ b/git/types.py @@ -73,19 +73,18 @@ See also the :class:`Tree_ish` and :class:`Commit_ish` unions. """ -Tree_ish = Union["Commit", "Tree"] -"""Union of :class:`~git.objects.base.Object`-based types that are inherently tree-ish. +Tree_ish = Union["Commit", "Tree", "TagObject"] +"""Union of :class:`~git.objects.base.Object`-based types that are sometimes tree-ish. See gitglossary(7) on "tree-ish": https://git-scm.com/docs/gitglossary#def_tree-ish :note: - This union comprises **only** the :class:`~git.objects.commit.Commit` and - :class:`~git.objects.tree.Tree` classes, **all** of whose instances are tree-ish. - This has been done because of the way GitPython uses it as a static type annotation. - - :class:`~git.objects.tag.TagObject`, some but not all of whose instances are - tree-ish (those representing git tag objects that ultimately resolve to a tree or - commit), is not covered as part of this union type. + :class:`~git.objects.tree.Tree` and :class:`~git.objects.commit.Commit` are the + classes whose instances are all tree-ish. This union includes them, but also + :class:`~git.objects.tag.TagObject`, only **some** of whose instances are tree-ish. + Whether a particular :class:`~git.objects.tag.TagObject` peels (recursively + dereferences) to a tree or commit, rather than a blob, can in general only be known + at runtime. :note: See also the :class:`AnyGitObject` union of all four classes corresponding to git @@ -102,12 +101,8 @@ commit-ish. This union type includes :class:`~git.objects.commit.Commit`, but also :class:`~git.objects.tag.TagObject`, only **some** of whose instances are commit-ish. Whether a particular :class:`~git.objects.tag.TagObject` peels - (recursively dereferences) to a commit can in general only be known at runtime. - -:note: - This is an inversion of the situation with :class:`Tree_ish`. This union is broader - than all commit-ish objects, while :class:`Tree_ish` is narrower than all tree-ish - objects. + (recursively dereferences) to a commit, rather than a tree or blob, can in general + only be known at runtime. :note: See also the :class:`AnyGitObject` union of all four classes corresponding to git From 0969db92133e7562038275086ae560bda5b9d6e4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 15 Mar 2024 12:30:22 -0400 Subject: [PATCH 0846/1392] Put Sphinx conf.py in the same style as other code This is mostly a change to comments. --- doc/source/conf.py | 87 ++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 45 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 9c22ca06a..e25c1e5dc 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -3,29 +3,28 @@ # # This file is execfile()d with the current directory set to its containing dir. # -# The contents of this file are pickled, so don't put values in the namespace -# that aren't pickleable (module imports are okay, they're removed automatically). +# The contents of this file are pickled, so don't put values in the namespace that +# aren't pickleable (module imports are okay, they're removed automatically). # -# Note that not all possible configuration values are present in this -# autogenerated file. +# Note that not all possible configuration values are present in this autogenerated +# file. # -# All configuration values have a default; values that are commented out -# serve to show the default. +# All configuration values have a default; values that are commented out serve to show +# the default. -import sys import os +import sys -# If your extensions are in another directory, add it here. If the directory -# is relative to the documentation root, use os.path.abspath to make it -# absolute, like shown here. +# If your extensions are in another directory, add it here. If the directory is relative +# to the documentation root, use os.path.abspath to make it absolute, like shown here. # sys.path.append(os.path.abspath('.')) sys.path.insert(0, os.path.abspath("../..")) # General configuration # --------------------- -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +# Add any Sphinx extension module names here, as strings. They can be extensions coming +# with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ["sphinx.ext.autodoc", "sphinx.ext.doctest"] # Add any paths that contain templates here, relative to this directory. @@ -44,9 +43,8 @@ project = "GitPython" copyright = "Copyright (C) 2008, 2009 Michael Trier and contributors, 2010-2015 Sebastian Thiel" -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. +# The version info for the project you're documenting, acts as replacement for |version| +# and |release|, also used in various other places throughout the built documents. # # The short X.Y version. with open(os.path.join(os.path.dirname(__file__), "..", "..", "VERSION")) as fd: @@ -55,8 +53,8 @@ # The full version, including alpha/beta/rc tags. release = VERSION -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. +# The language for content autogenerated by Sphinx. Refer to documentation for a list of +# supported languages. # language = None # There are two options for replacing |today|: either, you set today to some @@ -68,8 +66,8 @@ # List of documents that shouldn't be included in the build. # unused_docs = [] -# List of directories, relative to source directory, that shouldn't be searched -# for source files. +# List of directories, relative to source directory, that shouldn't be searched for +# source files. exclude_trees = ["build"] # The reST default role (used for this markup: `text`) to use for all documents. @@ -78,12 +76,12 @@ # If true, '()' will be appended to :func: etc. cross-reference text. # add_function_parentheses = True -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). +# If true, the current module name will be prepended to all description unit titles +# (such as .. function::). # add_module_names = True -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. +# If true, sectionauthor and moduleauthor directives will be shown in the output. +# They are ignored by default. # show_authors = False # The name of the Pygments (syntax highlighting) style to use. @@ -96,40 +94,39 @@ html_theme = "sphinx_rtd_theme" html_theme_options = {} -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". +# The name for this set of Sphinx documents. +# If None, it defaults to " v documentation". # html_title = None -# A shorter title for the navigation bar. Default is the same as html_title. +# A shorter title for the navigation bar. Default is the same as html_title. # html_short_title = None -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. +# The name of an image file (relative to this directory) to place at the top of the +# sidebar. # html_logo = None -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. +# The name of an image file (within the static path) to use as favicon of the docs. +# This file should be a Windows icon file (.ico) being 16x16 or 32x32 pixels large. # html_favicon = None -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". +# Add any paths that contain custom static files (such as style sheets) here, relative +# to this directory. They are copied after the builtin static files, so a file named +# "default.css" will overwrite the builtin "default.css". html_static_path = [] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, using the +# given strftime format. # html_last_updated_fmt = '%b %d, %Y' -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. +# If true, SmartyPants will be used to convert quotes and dashes to typographically +# correct entities. # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. # html_sidebars = {} -# Additional templates that should be rendered to pages, maps page names to -# template names. +# Additional templates that should be rendered to pages, maps page names to template +# names. # html_additional_pages = {} # If false, no module index is generated. @@ -144,9 +141,9 @@ # If true, the reST sources are included in the HTML build as _sources/. # html_copy_source = True -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. +# If true, an OpenSearch description file will be output, and all pages will contain a +# tag referring to it. The value of this option must be the base URL from which +# the finished HTML is served. # html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). @@ -171,8 +168,8 @@ ("index", "GitPython.tex", "GitPython Documentation", "Michael Trier", "manual"), ] -# The name of an image file (relative to this directory) to place at the top of -# the title page. +# The name of an image file (relative to this directory) to place at the top of the +# title page. # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, From c5a29a97e9305f86723f7fceb3a2cd5eb1dadfa6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 15 Mar 2024 12:35:49 -0400 Subject: [PATCH 0847/1392] Link Sphinx manpage references to online Git docs (This is grouped as a general option since it is technically classified as such, even though it currently only affects HTML output.) --- doc/source/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/conf.py b/doc/source/conf.py index e25c1e5dc..809762483 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -87,6 +87,8 @@ # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" +manpages_url = "https://git-scm.com/docs/{page}" + # Options for HTML output # ----------------------- From 7f1675d2f190c5471186af67d4e18107a4efb745 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 15 Mar 2024 14:34:29 -0400 Subject: [PATCH 0848/1392] Remove a spurious extra backtick from a docstring --- git/objects/submodule/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 0e51ae711..a5d1a6ad1 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -464,7 +464,7 @@ def add( :param url: git-clone compatible URL, see git-clone reference for more information. - If ``None```, the repository is assumed to exist, and the url of the first + If ``None``, the repository is assumed to exist, and the url of the first remote is taken instead. This is useful if you want to make an existing repository a submodule of another one. From e883293fd8e18ce2e1d2723aa395f3ff35e20213 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 15 Mar 2024 14:43:35 -0400 Subject: [PATCH 0849/1392] Add a missing Sphinx reference to a class --- git/repo/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/repo/base.py b/git/repo/base.py index fe01a9279..1238f0e17 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -1452,7 +1452,7 @@ def clone( :param kwargs: * ``odbt`` = ObjectDatabase Type, allowing to determine the object database - implementation used by the returned Repo instance. + implementation used by the returned :class:`Repo` instance. * All remaining keyword arguments are given to the ``git clone`` command. :return: From d8ab99c26ea0d60b8def5d0fd1e11b0750c47c37 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 15 Mar 2024 16:02:20 -0400 Subject: [PATCH 0850/1392] Use Sphinx manpage references where applicable In docstrings within the git module. This makes text of the same general form as, e.g. git-rev-parse or ``git rev-parse`` or URLs that link directly to a documentation page equivalent to a manpage or that link to the first section where preceding material is trivial... ...instead be in the form: :manpage:`git-rev-parse(1)` with variations as appropriate, for example changing gitglossary(7) to :manpage:`gitglossary(7)` and making other changes accordingly, such as adjusting phrasing and the use of hyphens in a small number of cases. Together with c5a29a9, which made such references linkify to the' official online documentation for Git, this makes it so that when git subcommands are mentioned in docstrings, the Sphinx autodoc generated documentation in the API Reference page now renders them as links to the relevant documentation page. Links to specific sections where it matters or potentially matters that it goes to that section are not replaced. In particular, links to specific entries in gitglossary(7) are not replaced. To do this properly would involve adding a new Sphinx role for it, which would work well in the rendered documentation but could be unclear when the documentation is read in docstrings appearing in the code. --- git/cmd.py | 9 +++--- git/config.py | 2 +- git/diff.py | 6 ++-- git/index/base.py | 55 ++++++++++++++++++----------------- git/index/typ.py | 3 +- git/objects/base.py | 2 +- git/objects/blob.py | 3 +- git/objects/commit.py | 23 ++++++++------- git/objects/submodule/base.py | 12 ++++---- git/objects/tag.py | 2 +- git/objects/tree.py | 2 +- git/refs/head.py | 2 +- git/refs/symbolic.py | 3 +- git/refs/tag.py | 2 +- git/remote.py | 20 ++++++------- git/repo/base.py | 31 +++++++++++--------- git/repo/fun.py | 6 ++-- git/types.py | 14 +++++---- git/util.py | 11 +++---- 19 files changed, 109 insertions(+), 99 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index ab2688a25..2862b1600 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -901,7 +901,8 @@ def working_dir(self) -> Union[None, PathLike]: def version_info(self) -> Tuple[int, ...]: """ :return: Tuple with integers representing the major, minor and additional - version numbers as parsed from ``git version``. Up to four fields are used. + version numbers as parsed from :manpage:`git-version(1)`. Up to four fields + are used. This value is generated on demand and is cached. """ @@ -1038,9 +1039,9 @@ def execute( 3. Deeper descendants do not receive signals, though they may sometimes terminate as a consequence of their parent processes being killed. 4. `kill_after_timeout` uses ``SIGKILL``, which can have negative side - effects on a repository. For example, stale locks in case of ``git gc`` - could render the repository incapable of accepting changes until the lock - is manually removed. + effects on a repository. For example, stale locks in case of + :manpage:`git-gc(1)` could render the repository incapable of accepting + changes until the lock is manually removed. :param with_stdout: If ``True``, default ``True``, we open stdout on the created process. diff --git a/git/config.py b/git/config.py index 4441c2187..f74d290cc 100644 --- a/git/config.py +++ b/git/config.py @@ -270,7 +270,7 @@ def get_config_path(config_level: Lit_config_levels) -> str: class GitConfigParser(cp.RawConfigParser, metaclass=MetaParserBuilder): """Implements specifics required to read git style configuration files. - This variation behaves much like the ``git config`` command, such that the + This variation behaves much like the :manpage:`git-config(1)` command, such that the configuration will be read on demand based on the filepath given during initialization. diff --git a/git/diff.py b/git/diff.py index 06935f87e..a6322ff57 100644 --- a/git/diff.py +++ b/git/diff.py @@ -222,8 +222,8 @@ def diff( to be read and diffed. :param kwargs: - Additional arguments passed to ``git diff``, such as ``R=True`` to swap both - sides of the diff. + Additional arguments passed to :manpage:`git-diff(1)`, such as ``R=True`` to + swap both sides of the diff. :return: A :class:`DiffIndex` representing the computed diff. @@ -590,7 +590,7 @@ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoIn The repository we are operating on. :param proc: - ``git diff`` process to read from + :manpage:`git-diff(1)` process to read from (supports :class:`Git.AutoInterrupt ` wrapper). :return: diff --git a/git/index/base.py b/git/index/base.py index 59b019f0f..cb67d2c29 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -225,11 +225,11 @@ def write( :param ignore_extension_data: If ``True``, the TREE type extension data read in the index will not be - written to disk. NOTE that no extension data is actually written. - Use this if you have altered the index and would like to use - ``git write-tree`` afterwards to create a tree representing your written - changes. If this data is present in the written index, ``git write-tree`` - will instead write the stored/cached tree. + written to disk. NOTE that no extension data is actually written. Use this + if you have altered the index and would like to use + :manpage:`git-write-tree(1)` afterwards to create a tree representing your + written changes. If this data is present in the written index, + :manpage:`git-write-tree(1)` will instead write the stored/cached tree. Alternatively, use :meth:`write_tree` to handle this case automatically. """ # Make sure we have our entries read before getting a write lock. @@ -343,7 +343,7 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile tree, tree 3 is the 'other' one. :param kwargs: - Additional arguments passed to ``git read-tree``. + Additional arguments passed to :manpage:`git-read-tree(1)`. :return: New :class:`IndexFile` instance. It will point to a temporary index location @@ -355,9 +355,9 @@ def from_tree(cls, repo: "Repo", *treeish: Treeish, **kwargs: Any) -> "IndexFile automatically resolve more cases in a commonly correct manner. Specify ``trivial=True`` as a keyword argument to override that. - As the underlying ``git read-tree`` command takes into account the current - index, it will be temporarily moved out of the way to prevent any unexpected - interference. + As the underlying :manpage:`git-read-tree(1)` command takes into account the + current index, it will be temporarily moved out of the way to prevent any + unexpected interference. """ if len(treeish) == 0 or len(treeish) > 3: raise ValueError("Please specify between 1 and 3 treeish, got %i" % len(treeish)) @@ -470,10 +470,10 @@ def _write_path_to_stdin( In that case, it will return ``None``. :note: - There is a bug in git-update-index that prevents it from sending reports - just in time. This is why we have a version that tries to read stdout and - one which doesn't. In fact, the stdout is not important as the piped-in - files are processed anyway and just in time. + There is a bug in :manpage:`git-update-index(1)` that prevents it from + sending reports just in time. This is why we have a version that tries to + read stdout and one which doesn't. In fact, the stdout is not important as + the piped-in files are processed anyway and just in time. :note: Newlines are essential here, git's behaviour is somewhat inconsistent on @@ -782,11 +782,12 @@ def add( directories like ``lib``, which will add all the files within the directory and subdirectories. - This equals a straight ``git add``. + This equals a straight :manpage:`git-add(1)`. They are added at stage 0. - - :class:~`git.objects.blob.Blob` or :class:`~git.objects.submodule.base.Submodule` object + - :class:~`git.objects.blob.Blob` or + :class:`~git.objects.submodule.base.Submodule` object Blobs are added as they are assuming a valid mode is set. @@ -818,8 +819,8 @@ def add( :param force: **CURRENTLY INEFFECTIVE** If ``True``, otherwise ignored or excluded files will be added anyway. As - opposed to the ``git add`` command, we enable this flag by default as the - API user usually wants the item to be added even though they might be + opposed to the :manpage:`git-add(1)` command, we enable this flag by default + as the API user usually wants the item to be added even though they might be excluded. :param fprogress: @@ -850,8 +851,8 @@ def add( :param write_extension_data: If ``True``, extension data will be written back to the index. This can lead to issues in case it is containing the 'TREE' extension, which will cause - the ``git commit`` command to write an old tree, instead of a new one - representing the now changed index. + the :manpage:`git-commit(1)` command to write an old tree, instead of a new + one representing the now changed index. This doesn't matter if you use :meth:`IndexFile.commit`, which ignores the 'TREE' extension altogether. You should set it to ``True`` if you intend to @@ -1008,8 +1009,8 @@ def remove( uncommitted changes in it. :param kwargs: - Additional keyword arguments to be passed to ``git rm``, such as ``r`` to - allow recursive removal. + Additional keyword arguments to be passed to :manpage:`git-rm(1)`, such as + ``r`` to allow recursive removal. :return: List(path_string, ...) list of repository relative paths that have been @@ -1058,7 +1059,7 @@ def move( skipped. :param kwargs: - Additional arguments you would like to pass to ``git mv``, such as + Additional arguments you would like to pass to :manpage:`git-mv(1)`, such as ``dry_run`` or ``force``. :return: @@ -1224,7 +1225,7 @@ def checkout( prior and after a file has been checked out. :param kwargs: - Additional arguments to be passed to ``git checkout-index``. + Additional arguments to be passed to :manpage:`git-checkout-index(1)`. :return: Iterable yielding paths to files which have been checked out and are @@ -1243,8 +1244,8 @@ def checkout( The checkout is limited to checking out the files in the index. Files which are not in the index anymore and exist in the working tree will not be deleted. This behaviour is fundamentally different to ``head.checkout``, - i.e. if you want ``git checkout`` like behaviour, use ``head.checkout`` - instead of ``index.checkout``. + i.e. if you want :manpage:`git-checkout(1)`-like behaviour, use + ``head.checkout`` instead of ``index.checkout``. """ args = ["--index"] if force: @@ -1416,14 +1417,14 @@ def reset( raised. :param kwargs: - Additional keyword arguments passed to ``git reset``. + Additional keyword arguments passed to :manpage:`git-reset(1)`. :note: :meth:`IndexFile.reset`, as opposed to :meth:`HEAD.reset `, will not delete any files in order to maintain a consistent working tree. Instead, it will just check out the files according to their state in the index. - If you want ``git reset``-like behaviour, use + If you want :manpage:`git-reset(1)`-like behaviour, use :meth:`HEAD.reset ` instead. :return: diff --git a/git/index/typ.py b/git/index/typ.py index c247fab99..ffd76dc46 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -130,8 +130,7 @@ def stage(self) -> int: * 3 = stage of entries from the 'right' side of the merge :note: - For more information, see: - http://www.kernel.org/pub/software/scm/git/docs/git-read-tree.html + For more information, see :manpage:`git-read-tree(1)`. """ return (self.flags & CE_STAGEMASK) >> CE_STAGESHIFT diff --git a/git/objects/base.py b/git/objects/base.py index f568a4bc5..22d939aa6 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -45,7 +45,7 @@ class Object(LazyMixin): * :class:`Commit ` * :class:`TagObject ` - See gitglossary(7) on: + See :manpage:`gitglossary(7)` on: * "object": https://git-scm.com/docs/gitglossary#def_object * "object type": https://git-scm.com/docs/gitglossary#def_object_type diff --git a/git/objects/blob.py b/git/objects/blob.py index b49930edf..122d5f731 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -19,7 +19,8 @@ class Blob(base.IndexObject): """A Blob encapsulates a git blob object. - See gitglossary(7) on "blob": https://git-scm.com/docs/gitglossary#def_blob_object + See :manpage:`gitglossary(7)` on "blob": + https://git-scm.com/docs/gitglossary#def_blob_object """ DEFAULT_MIME_TYPE = "text/plain" diff --git a/git/objects/commit.py b/git/objects/commit.py index 473eae8cc..3c5d8fba3 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -65,7 +65,7 @@ class Commit(base.Object, TraversableIterableObj, Diffable, Serializable): """Wraps a git commit object. - See gitglossary(7) on "commit object": + See :manpage:`gitglossary(7)` on "commit object": https://git-scm.com/docs/gitglossary#def_commit_object :note: @@ -269,8 +269,9 @@ def count(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) actually containing the paths. :param kwargs: - Additional options to be passed to ``git rev-list``. They must not alter the - output style of the command, or parsing will yield incorrect results. + Additional options to be passed to :manpage:`git-rev-list(1)`. They must not + alter the output style of the command, or parsing will yield incorrect + results. :return: An int defining the number of reachable commits @@ -307,14 +308,14 @@ def iter_items( The :class:`~git.repo.base.Repo`. :param rev: - Revision specifier. See git-rev-parse for viable options. + Revision specifier. See :manpage:`git-rev-parse(1)` for viable options. :param paths: An optional path or list of paths. If set only :class:`Commit`\s that include the path or paths will be considered. :param kwargs: - Optional keyword arguments to ``git rev-list`` where: + Optional keyword arguments to :manpage:`git-rev-list(1)` where: * ``max_count`` is the maximum number of commits to fetch. * ``skip`` is the number of commits to skip. @@ -353,7 +354,7 @@ def iter_parents(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs contain at least one of the paths. :param kwargs: - All arguments allowed by ``git rev-list``. + All arguments allowed by :manpage:`git-rev-list(1)`. :return: Iterator yielding :class:`Commit` objects which are parents of ``self`` @@ -404,7 +405,7 @@ def trailers_list(self) -> List[Tuple[str, str]]: """Get the trailers of the message as a list. Git messages can contain trailer information that are similar to RFC 822 e-mail - headers (see: https://git-scm.com/docs/git-interpret-trailers). + headers. See :manpage:`git-interpret-trailers(1)`. This function calls ``git interpret-trailers --parse`` onto the message to extract the trailer information, returns the raw trailer data as a list. @@ -456,7 +457,7 @@ def trailers_dict(self) -> Dict[str, List[str]]: """Get the trailers of the message as a dictionary. Git messages can contain trailer information that are similar to RFC 822 e-mail - headers (see: https://git-scm.com/docs/git-interpret-trailers). + headers. See :manpage:`git-interpret-trailers(1)`. This function calls ``git interpret-trailers --parse`` onto the message to extract the trailer information. The key value pairs are stripped of leading and @@ -499,7 +500,7 @@ def _iter_from_process_or_stream(cls, repo: "Repo", proc_or_stream: Union[Popen, from our lighting fast object database. :param proc: - ``git rev-list`` process instance - one sha per line. + :manpage:`git-rev-list(1)` process instance - one sha per line. :return: Iterator supplying :class:`Commit` objects @@ -596,8 +597,8 @@ def create_from_tree( :note: Additional information about the committer and author are taken from the - environment or from the git configuration. See git-commit-tree for more - information. + environment or from the git configuration. See :manpage:`git-commit-tree(1)` + for more information. """ if parent_commits is None: try: diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index a5d1a6ad1..d01aa448f 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -342,7 +342,7 @@ def _clone_repo( Allow unsafe options to be used, like ``--upload-pack``. :param kwargs: - Additional arguments given to ``git clone``. + Additional arguments given to :manpage:`git-clone(1)`. """ module_abspath = cls._module_abspath(repo, path, name) module_checkout_path = module_abspath @@ -463,10 +463,10 @@ def add( It will be created as required during the repository initialization. :param url: - git-clone compatible URL, see git-clone reference for more information. - If ``None``, the repository is assumed to exist, and the url of the first - remote is taken instead. This is useful if you want to make an existing - repository a submodule of another one. + ``git clone ...``-compatible URL. See :manpage:`git-clone(1)` for more + information. If ``None``, the repository is assumed to exist, and the URL of + the first remote is taken instead. This is useful if you want to make an + existing repository a submodule of another one. :param branch: Name of branch at which the submodule should (later) be checked out. The @@ -696,7 +696,7 @@ def update( its value. :param clone_multi_options: - List of ``git clone`` options. + List of :manpage:`git-clone(1)` options. Please see :meth:`Repo.clone ` for details. They only take effect with the `init` option. diff --git a/git/objects/tag.py b/git/objects/tag.py index e7ecfa62b..52d79751f 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -37,7 +37,7 @@ class TagObject(base.Object): """Annotated (i.e. non-lightweight) tag carrying additional information about an object we are pointing to. - See gitglossary(7) on "tag object": + See :manpage:`gitglossary(7)` on "tag object": https://git-scm.com/docs/gitglossary#def_tag_object """ diff --git a/git/objects/tree.py b/git/objects/tree.py index 308dd47a0..ad67f8e47 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -168,7 +168,7 @@ class Tree(IndexObject, git_diff.Diffable, util.Traversable, util.Serializable): R"""Tree objects represent an ordered list of :class:`~git.objects.blob.Blob`\s and other :class:`Tree`\s. - See gitglossary(7) on "tree object": + See :manpage:`gitglossary(7)` on "tree object": https://git-scm.com/docs/gitglossary#def_tree_object Subscripting is supported, as with a list or dict: diff --git a/git/refs/head.py b/git/refs/head.py index aae5767d4..b65189621 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -89,7 +89,7 @@ def reset( that are to be reset. This allows to partially reset individual files. :param kwargs: - Additional arguments passed to ``git reset``. + Additional arguments passed to :manpage:`git-reset(1)`. :return: self diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 754e90089..dea047d83 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -173,8 +173,7 @@ def dereference_recursive(cls, repo: "Repo", ref_path: Union[PathLike, None]) -> def _check_ref_name_valid(ref_path: PathLike) -> None: """Check a ref name for validity. - This is based on the rules described in: - https://git-scm.com/docs/git-check-ref-format/#_description + This is based on the rules described in :manpage:`git-check-ref-format(1)`. """ previous: Union[str, None] = None one_before_previous: Union[str, None] = None diff --git a/git/refs/tag.py b/git/refs/tag.py index a1d0b470f..f653d4e7d 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -124,7 +124,7 @@ def create( If ``True``, force creation of a tag even though that tag already exists. :param kwargs: - Additional keyword arguments to be passed to ``git tag``. + Additional keyword arguments to be passed to :manpage:`git-tag(1)`. :return: A new :class:`TagReference`. diff --git a/git/remote.py b/git/remote.py index 1723216a4..ac58a4f18 100644 --- a/git/remote.py +++ b/git/remote.py @@ -333,8 +333,8 @@ class FetchInfo(IterableObj): @classmethod def refresh(cls) -> Literal[True]: - """Update information about which ``git fetch`` flags are supported by the git - executable being used. + """Update information about which :manpage:`git-fetch(1)` flags are supported + by the git executable being used. Called by the :func:`git.refresh` function in the top level ``__init__``. """ @@ -1015,13 +1015,13 @@ def fetch( "grab the master branch head from the $URL and store it as my origin branch head". And ``git push $URL refs/heads/master:refs/heads/to-upstream`` means "publish my master branch head as to-upstream branch at $URL". - See also git-push(1). + See also :manpage:`git-push(1)`. - Taken from the git manual, gitglossary(7). + Taken from the git manual, :manpage:`gitglossary(7)`. - Fetch supports multiple refspecs (as the underlying git-fetch does) - - supplying a list rather than a string for 'refspec' will make use of this - facility. + Fetch supports multiple refspecs (as the underlying :manpage:`git-fetch(1)` + does) - supplying a list rather than a string for 'refspec' will make use of + this facility. :param progress: See the :meth:`push` method. @@ -1040,7 +1040,7 @@ def fetch( Allow unsafe options to be used, like ``--upload-pack``. :param kwargs: - Additional arguments to be passed to ``git fetch``. + Additional arguments to be passed to :manpage:`git-fetch(1)`. :return: IterableList(FetchInfo, ...) list of :class:`FetchInfo` instances providing @@ -1104,7 +1104,7 @@ def pull( Allow unsafe options to be used, like ``--upload-pack``. :param kwargs: - Additional arguments to be passed to ``git pull``. + Additional arguments to be passed to :manpage:`git-pull(1)`. :return: Please see :meth:`fetch` method. @@ -1170,7 +1170,7 @@ def push( Allow unsafe options to be used, like ``--receive-pack``. :param kwargs: - Additional arguments to be passed to ``git push``. + Additional arguments to be passed to :manpage:`git-push(1)`. :return: A :class:`PushInfoList` object, where each list member represents an diff --git a/git/repo/base.py b/git/repo/base.py index 1238f0e17..d6a161305 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -149,7 +149,7 @@ class Repo: "--config", "-c", ] - """Options to ``git clone`` that allow arbitrary commands to be executed. + """Options to :manpage:`git-clone(1)` that allow arbitrary commands to be executed. The ``--upload-pack``/``-u`` option allows users to execute arbitrary commands directly: @@ -572,7 +572,7 @@ def delete_head(self, *heads: "Union[str, Head]", **kwargs: Any) -> None: """Delete the given heads. :param kwargs: - Additional keyword arguments to be passed to ``git branch``. + Additional keyword arguments to be passed to :manpage:`git-branch(1)`. """ return Head.delete(self, *heads, **kwargs) @@ -700,7 +700,7 @@ def commit(self, rev: Union[str, Commit_ish, None] = None) -> Commit: """The :class:`~git.objects.commit.Commit` object for the specified revision. :param rev: - Revision specifier, see ``git rev-parse`` for viable options. + Revision specifier, see :manpage:`git-rev-parse(1)` for viable options. :return: :class:`~git.objects.commit.Commit` @@ -749,7 +749,7 @@ def iter_commits( history of a given ref/commit. :param rev: - Revision specifier, see ``git rev-parse`` for viable options. + Revision specifier, see :manpage:`git-rev-parse(1)` for viable options. If ``None``, the active branch will be used. :param paths: @@ -757,7 +757,7 @@ def iter_commits( path or paths will be returned. :param kwargs: - Arguments to be passed to ``git rev-list``. + Arguments to be passed to :manpage:`git-rev-list(1)`. Common ones are ``max_count`` and ``skip``. :note: @@ -930,8 +930,8 @@ def is_dirty( """ :return: ``True`` if the repository is considered dirty. By default it will react - like a git-status without untracked files, hence it is dirty if the index or - the working copy have changes. + like a :manpage:`git-status(1)` without untracked files, hence it is dirty + if the index or the working copy have changes. """ if self._bare: # Bare repositories with no associated working directory are @@ -1001,7 +1001,7 @@ def _get_untracked_files(self, *args: Any, **kwargs: Any) -> List[str]: def ignored(self, *paths: PathLike) -> List[str]: """Checks if paths are ignored via ``.gitignore``. - This does so using the ``git check-ignore`` method. + This does so using the :manpage:`git-check-ignore(1)` method. :param paths: List of paths to check whether they are ignored or not. @@ -1044,7 +1044,7 @@ def blame_incremental(self, rev: str | HEAD | None, file: str, **kwargs: Any) -> :param rev: Revision specifier. If ``None``, the blame will include all the latest uncommitted changes. Otherwise, anything successfully parsed by - ``git rev-parse`` is a valid option. + :manpage:`git-rev-parse(1)` is a valid option. :return: Lazy iterator of :class:`BlameEntry` tuples, where the commit indicates the @@ -1140,7 +1140,7 @@ def blame( :param rev: Revision specifier. If ``None``, the blame will include all the latest uncommitted changes. Otherwise, anything successfully parsed by - ``git rev-parse`` is a valid option. + :manpage:`git-rev-parse(1)` is a valid option. :return: list: [git.Commit, list: []] @@ -1312,7 +1312,8 @@ def init( environment variables. :param kwargs: - Keyword arguments serving as additional options to the ``git init`` command. + Keyword arguments serving as additional options to the + :manpage:`git-init(1)` command. :return: :class:`Repo` (the newly created repo) @@ -1432,7 +1433,8 @@ def clone( See :meth:`Remote.push `. :param multi_options: - A list of ``git clone`` options that can be provided multiple times. + A list of :manpage:`git-clone(1)` options that can be provided multiple + times. One option per list item which is passed exactly as specified to clone. For example:: @@ -1453,7 +1455,8 @@ def clone( :param kwargs: * ``odbt`` = ObjectDatabase Type, allowing to determine the object database implementation used by the returned :class:`Repo` instance. - * All remaining keyword arguments are given to the ``git clone`` command. + * All remaining keyword arguments are given to the :manpage:`git-clone(1)` + command. :return: :class:`Repo` (the newly cloned repo) @@ -1551,7 +1554,7 @@ def archive( The optional prefix to prepend to each filename in the archive. :param kwargs: - Additional arguments passed to ``git archive``: + Additional arguments passed to :manpage:`git-archive(1)`: * Use the ``format`` argument to define the kind of format. Use specialized ostreams to write any format supported by Python. diff --git a/git/repo/fun.py b/git/repo/fun.py index 0ac481206..13c85ea22 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -226,7 +226,7 @@ def to_commit(obj: Object) -> "Commit": def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: - """Parse a revision string. Like ``git rev-parse``. + """Parse a revision string. Like :manpage:`git-rev-parse(1)`. :return: `~git.objects.base.Object` at the given revision. @@ -239,8 +239,8 @@ def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: * :class:`Blob ` :param rev: - ``git rev-parse``-compatible revision specification as string. Please see - http://www.kernel.org/pub/software/scm/git/docs/git-rev-parse.html for details. + :manpage:`git-rev-parse(1)`-compatible revision specification as string. + Please see :manpage:`git-rev-parse(1)` for details. :raise gitdb.exc.BadObject: If the given revision could not be found. diff --git a/git/types.py b/git/types.py index 64f7629dc..a30179a25 100644 --- a/git/types.py +++ b/git/types.py @@ -57,7 +57,8 @@ * :class:`Commit ` * :class:`TagObject ` -Those GitPython classes represent the four git object types, per gitglossary(7): +Those GitPython classes represent the four git object types, per +:manpage:`gitglossary(7)`: * "blob": https://git-scm.com/docs/gitglossary#def_blob_object * "tree object": https://git-scm.com/docs/gitglossary#def_tree_object @@ -76,7 +77,8 @@ Tree_ish = Union["Commit", "Tree", "TagObject"] """Union of :class:`~git.objects.base.Object`-based types that are sometimes tree-ish. -See gitglossary(7) on "tree-ish": https://git-scm.com/docs/gitglossary#def_tree-ish +See :manpage:`gitglossary(7)` on "tree-ish": +https://git-scm.com/docs/gitglossary#def_tree-ish :note: :class:`~git.objects.tree.Tree` and :class:`~git.objects.commit.Commit` are the @@ -94,7 +96,8 @@ Commit_ish = Union["Commit", "TagObject"] """Union of :class:`~git.objects.base.Object`-based types that are sometimes commit-ish. -See gitglossary(7) on "commit-ish": https://git-scm.com/docs/gitglossary#def_commit-ish +See :manpage:`gitglossary(7)` on "commit-ish": +https://git-scm.com/docs/gitglossary#def_commit-ish :note: :class:`~git.objects.commit.Commit` is the only class whose instances are all @@ -117,8 +120,9 @@ values in :class:`~git.objects.base.Object` subclasses that represent git objects. These literals therefore correspond to the types in the :class:`AnyGitObject` union. -These are the same strings git itself uses to identify its four object types. See -gitglossary(7) on "object type": https://git-scm.com/docs/gitglossary#def_object_type +These are the same strings git itself uses to identify its four object types. +See :manpage:`gitglossary(7)` on "object type": +https://git-scm.com/docs/gitglossary#def_object_type """ Lit_commit_ish = Literal["commit", "tag"] diff --git a/git/util.py b/git/util.py index 2a9dd10a9..bef828dbf 100644 --- a/git/util.py +++ b/git/util.py @@ -558,8 +558,8 @@ def remove_password_if_present(cmdline: Sequence[str]) -> List[str]: class RemoteProgress: """Handler providing an interface to parse progress information emitted by - ``git push`` and ``git fetch`` and to dispatch callbacks allowing subclasses to - react to the progress.""" + :manpage:`git-push(1)` and :manpage:`git-fetch(1)` and to dispatch callbacks + allowing subclasses to react to the progress.""" _num_op_codes: int = 9 ( @@ -595,8 +595,8 @@ def __init__(self) -> None: self.other_lines: List[str] = [] def _parse_progress_line(self, line: AnyStr) -> None: - """Parse progress information from the given line as retrieved by ``git push`` - or ``git fetch``. + """Parse progress information from the given line as retrieved by + :manpage:`git-push(1)` or :manpage:`git-fetch(1)`. - Lines that do not contain progress info are stored in :attr:`other_lines`. - Lines that seem to contain an error (i.e. start with ``error:`` or ``fatal:``) @@ -922,7 +922,8 @@ def __init__(self, total: Total_TD, files: Dict[PathLike, Files_TD]) -> None: @classmethod def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": - """Create a :class:`Stats` object from output retrieved by ``git diff``. + """Create a :class:`Stats` object from output retrieved by + :manpage:`git-diff(1)`. :return: :class:`git.Stats` From d271a84a67ab1cbb9c5922244fd5e17517520593 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 15 Mar 2024 16:14:28 -0400 Subject: [PATCH 0851/1392] Use more official link to index-format documentation This could also be written automatically by labeling it with the :manpage: role, but it doesn't seem to be an actual manpage, so I have not done that. --- git/index/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/index/base.py b/git/index/base.py index cb67d2c29..dfb687260 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -862,7 +862,7 @@ def add( handled manually at all. All current built-in extensions are listed here: - http://opensource.apple.com/source/Git/Git-26/src/git-htmldocs/technical/index-format.txt + https://git-scm.com/docs/index-format :return: List of :class:`~git.index.typ.BaseIndexEntry`\s representing the entries From ca95c420ce5ed45fe33381b25dd1a1c65e150d40 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 15 Mar 2024 16:18:21 -0400 Subject: [PATCH 0852/1392] Use current main official link to git-clone URLS doc I am still not using the :manpage: role for this because it points to a specific section, which that role does not support, and it is important that it go to that specific section. (See d8ab99c for details.) --- git/repo/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index d6a161305..ce164274e 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -1488,8 +1488,7 @@ def clone_from( """Create a clone from the given URL. :param url: - Valid git url, see: - http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS + Valid git url, see: https://git-scm.com/docs/git-clone#URLS :param to_path: Path to which the repository should be cloned to. From 85434571090a24ac84d1ce81f3fa74909e3d5c5f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 15 Mar 2024 16:24:57 -0400 Subject: [PATCH 0853/1392] Use :const: for constants that had the :attr: role Note that this intentionally does *not* include some all-caps class attributes that are not really constants: - Git.GIT_PYTHON_GIT_EXECUTABLE is set by refresh functions, including on subsequent calls, which is an important and documented part of what those functions do. (Also, it is set automatically from an enviroment variable, which is not constant across runs; that it could not even in principle be replaced by a specific literal value is a further reason it is not a constant.) - The Git.USE_SHELL attribute is a more ambiguous case. It is given a literal value (False) which it preferably remains. But setting it has been documented as something users can do (in the changelog, and later in regard to setting it being a deprecated operation). The first of these is decisively not a constant even under a loose definition, while the second is less clear. I've kept both with the :attr: role. --- git/remote.py | 2 +- git/util.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/git/remote.py b/git/remote.py index ac58a4f18..2c452022e 100644 --- a/git/remote.py +++ b/git/remote.py @@ -1177,7 +1177,7 @@ def push( individual head which had been updated on the remote side. If the push contains rejected heads, these will have the - :attr:`PushInfo.ERROR` bit set in their flags. + :const:`PushInfo.ERROR` bit set in their flags. If the operation fails completely, the length of the returned :class:`PushInfoList` will be 0. diff --git a/git/util.py b/git/util.py index bef828dbf..0de724ceb 100644 --- a/git/util.py +++ b/git/util.py @@ -713,14 +713,14 @@ def update( :param op_code: Integer allowing to be compared against Operation IDs and stage IDs. - Stage IDs are :attr:`BEGIN` and :attr:`END`. :attr:`BEGIN` will only be set - once for each Operation ID as well as :attr:`END`. It may be that - :attr:`BEGIN` and :attr:`END` are set at once in case only one progress - message was emitted due to the speed of the operation. Between :attr:`BEGIN` - and :attr:`END`, none of these flags will be set. + Stage IDs are :const:`BEGIN` and :const:`END`. :const:`BEGIN` will only be + set once for each Operation ID as well as :const:`END`. It may be that + :const:`BEGIN` and :const:`END` are set at once in case only one progress + message was emitted due to the speed of the operation. Between + :const:`BEGIN` and :const:`END`, none of these flags will be set. - Operation IDs are all held within the :attr:`OP_MASK`. Only one Operation ID - will be active per call. + Operation IDs are all held within the :const:`OP_MASK`. Only one Operation + ID will be active per call. :param cur_count: Current absolute count of items. @@ -730,7 +730,7 @@ def update( maximum number of items or if it is (yet) unknown. :param message: - In case of the :attr:`WRITING` operation, it contains the amount of bytes + In case of the :const:`WRITING` operation, it contains the amount of bytes transferred. It may possibly be used for other purposes as well. :note: From 6626117259b93d0a3085dd17ed71ef1acd08c7a5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 16 Mar 2024 01:58:30 -0400 Subject: [PATCH 0854/1392] Fix parse_date docstring spacing; use RFC role There are only a three cases where a specific RFC is mentioned. Only one is in parse_date and the others are elsewhere. --- git/objects/commit.py | 8 ++++---- git/objects/util.py | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 3c5d8fba3..6a60c30bd 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -404,8 +404,8 @@ def trailers(self) -> Dict[str, str]: def trailers_list(self) -> List[Tuple[str, str]]: """Get the trailers of the message as a list. - Git messages can contain trailer information that are similar to RFC 822 e-mail - headers. See :manpage:`git-interpret-trailers(1)`. + Git messages can contain trailer information that are similar to :rfc:`822` + e-mail headers. See :manpage:`git-interpret-trailers(1)`. This function calls ``git interpret-trailers --parse`` onto the message to extract the trailer information, returns the raw trailer data as a list. @@ -456,8 +456,8 @@ def trailers_list(self) -> List[Tuple[str, str]]: def trailers_dict(self) -> Dict[str, List[str]]: """Get the trailers of the message as a dictionary. - Git messages can contain trailer information that are similar to RFC 822 e-mail - headers. See :manpage:`git-interpret-trailers(1)`. + Git messages can contain trailer information that are similar to :rfc:`822` + e-mail headers. See :manpage:`git-interpret-trailers(1)`. This function calls ``git interpret-trailers --parse`` onto the message to extract the trailer information. The key value pairs are stripped of leading and diff --git a/git/objects/util.py b/git/objects/util.py index 7cca05134..297b33b70 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -223,9 +223,8 @@ def parse_date(string_date: Union[str, datetime]) -> Tuple[int, int]: * Aware datetime instance * Git internal format: timestamp offset - * RFC 2822: ``Thu, 07 Apr 2005 22:13:13 +0200`` - * ISO 8601: ``2005-04-07T22:13:13`` - The T can be a space as well. + * :rfc:`2822`: ``Thu, 07 Apr 2005 22:13:13 +0200`` + * ISO 8601: ``2005-04-07T22:13:13`` - The ``T`` can be a space as well. :return: Tuple(int(timestamp_UTC), int(offset)), both in seconds since epoch From a957ae7a94ebede485e94d03da0258a0d2ad79e0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 16 Mar 2024 02:19:39 -0400 Subject: [PATCH 0855/1392] Use the :exc: role for exceptions Rather than the more general :class: role that was used for them. --- git/db.py | 2 +- git/index/base.py | 10 +++++----- git/objects/submodule/util.py | 2 +- git/objects/tree.py | 4 ++-- git/refs/head.py | 2 +- git/refs/reference.py | 2 +- git/refs/remote.py | 2 +- git/refs/symbolic.py | 2 +- git/repo/fun.py | 4 ++-- git/types.py | 2 +- git/util.py | 8 ++++---- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/git/db.py b/git/db.py index 15e791e6c..5b2ca4de2 100644 --- a/git/db.py +++ b/git/db.py @@ -59,7 +59,7 @@ def partial_to_complete_sha_hex(self, partial_hexsha: str) -> bytes: :raise gitdb.exc.BadObject: :note: - Currently we only raise :class:`~gitdb.exc.BadObject` as git does not + Currently we only raise :exc:`~gitdb.exc.BadObject` as git does not communicate ambiguous objects separately. """ try: diff --git a/git/index/base.py b/git/index/base.py index dfb687260..fb91e092c 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -550,8 +550,8 @@ def resolve_blobs(self, iter_blobs: Iterator[Blob]) -> "IndexFile": This will effectively remove the index entries of the respective path at all non-null stages and add the given blob as new stage null blob. - For each path there may only be one blob, otherwise a :class:`ValueError` will - be raised claiming the path is already at stage 0. + For each path there may only be one blob, otherwise a :exc:`ValueError` will be + raised claiming the path is already at stage 0. :raise ValueError: If one of the blobs already existed at stage 0. @@ -644,8 +644,8 @@ def _process_diff_args( def _to_relative_path(self, path: PathLike) -> PathLike: """ :return: - Version of path relative to our git directory or raise :class:`ValueError` - if it is not within our git directory. + Version of path relative to our git directory or raise :exc:`ValueError` if + it is not within our git directory. :raise ValueError: """ @@ -1215,7 +1215,7 @@ def checkout( :param force: If ``True``, existing files will be overwritten even if they contain local modifications. - If ``False``, these will trigger a :class:`~git.exc.CheckoutError`. + If ``False``, these will trigger a :exc:`~git.exc.CheckoutError`. :param fprogress: See :meth:`IndexFile.add` for signature and explanation. diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index e5af5eb31..10b994e9b 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -52,7 +52,7 @@ def mkhead(repo: "Repo", path: PathLike) -> "Head": def find_first_remote_branch(remotes: Sequence["Remote"], branch_name: str) -> "RemoteReference": """Find the remote branch matching the name of the given branch or raise - :class:`~git.exc.InvalidGitRepositoryError`.""" + :exc:`~git.exc.InvalidGitRepositoryError`.""" for remote in remotes: try: return remote.refs[branch_name] diff --git a/git/objects/tree.py b/git/objects/tree.py index ad67f8e47..c74df58a9 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -100,8 +100,8 @@ def add(self, sha: bytes, mode: int, name: str, force: bool = False) -> "TreeMod """Add the given item to the tree. If an item with the given name already exists, nothing will be done, but a - :class:`ValueError` will be raised if the sha and mode of the existing item do - not match the one you add, unless `force` is ``True``. + :exc:`ValueError` will be raised if the sha and mode of the existing item do not + match the one you add, unless `force` is ``True``. :param sha: The 20 or 40 byte sha of the item to add. diff --git a/git/refs/head.py b/git/refs/head.py index b65189621..86321d9ea 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -247,7 +247,7 @@ def checkout(self, force: bool = False, **kwargs: Any) -> Union["HEAD", "Head"]: :param force: If ``True``, changes to the index and the working tree will be discarded. - If ``False``, :class:`~git.exc.GitCommandError` will be raised in that + If ``False``, :exc:`~git.exc.GitCommandError` will be raised in that situation. :param kwargs: diff --git a/git/refs/reference.py b/git/refs/reference.py index cf418aa5d..62fb58420 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -21,7 +21,7 @@ def require_remote_ref_path(func: Callable[..., _T]) -> Callable[..., _T]: - """A decorator raising :class:`ValueError` if we are not a valid remote, based on the + """A decorator raising :exc:`ValueError` if we are not a valid remote, based on the path.""" def wrapper(self: T_References, *args: Any) -> _T: diff --git a/git/refs/remote.py b/git/refs/remote.py index 5cbd1b81b..3f9c6c6be 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -76,5 +76,5 @@ def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: @classmethod def create(cls, *args: Any, **kwargs: Any) -> NoReturn: - """Raise :class:`TypeError`. Defined so the ``create`` method is disabled.""" + """Raise :exc:`TypeError`. Defined so the ``create`` method is disabled.""" raise TypeError("Cannot explicitly create remote references") diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index dea047d83..2701f9f2b 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -738,7 +738,7 @@ def create( :param force: If ``True``, force creation even if a symbolic reference with that name - already exists. Raise :class:`OSError` otherwise. + already exists. Raise :exc:`OSError` otherwise. :param logmsg: If not ``None``, the message to append to the reflog. diff --git a/git/repo/fun.py b/git/repo/fun.py index 13c85ea22..738e5816d 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -153,8 +153,8 @@ def name_to_object(repo: "Repo", name: str, return_ref: bool = False) -> Union[A :param return_ref: If ``True``, and name specifies a reference, we will return the reference - instead of the object. Otherwise it will raise :class:`~gitdb.exc.BadObject` or - :class:`~gitdb.exc.BadName`. + instead of the object. Otherwise it will raise :exc:`~gitdb.exc.BadObject` or + :exc:`~gitdb.exc.BadName`. """ hexsha: Union[None, str, bytes] = None diff --git a/git/types.py b/git/types.py index a30179a25..e3ae9e3d5 100644 --- a/git/types.py +++ b/git/types.py @@ -190,7 +190,7 @@ def assert_never(inp: NoReturn, raise_error: bool = True, exc: Union[Exception, mismatch and cause a mypy error. :param raise_error: - If ``True``, will also raise :class:`ValueError` with a general "unhandled + If ``True``, will also raise :exc:`ValueError` with a general "unhandled literal" message, or the exception object passed as `exc`. :param exc: diff --git a/git/util.py b/git/util.py index 0de724ceb..37986edaa 100644 --- a/git/util.py +++ b/git/util.py @@ -158,8 +158,8 @@ def _read_win_env_flag(name: str, default: bool) -> bool: def unbare_repo(func: Callable[..., T]) -> Callable[..., T]: - """Methods with this decorator raise - :class:`~git.exc.InvalidGitRepositoryError` if they encounter a bare repository.""" + """Methods with this decorator raise :exc:`~git.exc.InvalidGitRepositoryError` if + they encounter a bare repository.""" from .exc import InvalidGitRepositoryError @@ -1094,8 +1094,8 @@ def __init__( self._max_block_time = max_block_time_s def _obtain_lock(self) -> None: - """This method blocks until it obtained the lock, or raises :class:`IOError` if - it ran out of time or if the parent directory was not available anymore. + """This method blocks until it obtained the lock, or raises :exc:`IOError` if it + ran out of time or if the parent directory was not available anymore. If this method returns, you are guaranteed to own the lock. """ From 1e5a9449573bfe70987dcaf44e120895131e9c66 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 Mar 2024 19:25:34 -0400 Subject: [PATCH 0856/1392] Add a script to validate refactored imports This script can be removed after imports are refactored and checked to see that module contents are the same as before or otherwise non-broken. This script assumes that module contents are the same as the contents of their dictionaries, and that all modules in the project get imported as a consequence of importing the top-level module. These are both the case currently for GitPython, but they do not hold for all projects, and may not hold for GitPython at some point in the future. --- .gitignore | 4 ++++ modattrs.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100755 modattrs.py diff --git a/.gitignore b/.gitignore index 7765293d8..1be4f3201 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,7 @@ output.txt # Finder metadata .DS_Store + +# Output files for modattrs.py (these entries will be removed soon) +a +b diff --git a/modattrs.py b/modattrs.py new file mode 100755 index 000000000..245f68912 --- /dev/null +++ b/modattrs.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +"""Script to get the names and "stabilized" reprs of module attributes in GitPython. + +Run with :envvar:`PYTHONHASHSEED` set to ``0`` for fully comparable results. These are +only still meaningful for comparing if the same platform and Python version are used. + +The output of this script should probably not be committed, because within the reprs of +objects found in modules, it may contain sensitive information, such as API keys stored +in environment variables. The "sanitization" performed here is only for common forms of +whitespace that clash with the output format. +""" + +# fmt: off + +__all__ = ["git", "main"] + +import itertools +import re +import sys + +import git + + +def main(): + # This assumes `import git` causes all of them to be loaded. + gitpython_modules = sorted( + (module_name, module) + for module_name, module in sys.modules.items() + if re.match(r"git(?:\.|$)", module_name) + ) + + # We will print two blank lines between successive module reports. + separators = itertools.chain(("",), itertools.repeat("\n\n")) + + # Report each module's contents. + for (module_name, module), separator in zip(gitpython_modules, separators): + print(f"{separator}{module_name}:") + + attributes = sorted( + (name, value) + for name, value in module.__dict__.items() + if name != "__all__" # Because we are deliberately adding these. + ) + + for name, value in attributes: + sanitized_repr = re.sub(r"[\r\n\v\f]", "?", repr(value)) + normalized_repr = re.sub(r" at 0x[0-9a-fA-F]+", " at 0x...", sanitized_repr) + print(f" {name}: {normalized_repr}") + + +if __name__ == "__main__": + main() From 5b2771d23af2156fb7e12ee6406bffbabcb9e95d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 10:59:37 -0400 Subject: [PATCH 0857/1392] Add regression tests of the git.util aliasing situation Although this situation is not inherently desirable, for backward compatibility it cannot change at this time. It may be possible to change it in the next major version of GitPython, but even then it should not be changed accidentally, which can easily happen while refactoring imports. This tests the highest-risk accidental change (of those that are currently known) of the kind that the temporary modattrs.py script exists to help safeguard against. That script will be removed when the immediately forthcoming import refactoring is complete, whereas these test cases can be kept. For information about the specific situation this helps ensure isn't changed accidentally, see the new test cases' docstrings, as well as the next commit (which will test modattrs.py and these test cases by performing an incomplete change that would be a bug until completed). This commit adds three test cases. The first tests the unintuitive aspect of the current situation: - test_git_util_attribute_is_git_index_util The other two test the intuitive aspects of it, i.e., they test that changes (perhaps in an attempt to preserve the aspect needed for backward compatibility) do not make `git.util` unusual in new (and themselves incompatible) ways: - test_git_index_util_attribute_is_git_index_util - test_separate_git_util_module_exists The latter tests should also clarify, for readers of the tests, the limited nature of the condition the first test asserts. --- test/test_imports.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 test/test_imports.py diff --git a/test/test_imports.py b/test/test_imports.py new file mode 100644 index 000000000..8e70c6689 --- /dev/null +++ b/test/test_imports.py @@ -0,0 +1,32 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +import sys + +import git + + +def test_git_util_attribute_is_git_index_util(): + """The top-level module's ``util`` attribute is really :mod:`git.index.util`. + + Although this situation is unintuitive and not a design goal, this has historically + been the case, and it should not be changed without considering the effect on + backward compatibility. In practice, it cannot be changed at least until the next + major version of GitPython. This test checks that it is not accidentally changed, + which could happen when refactoring imports. + """ + assert git.util is git.index.util + + +def test_git_index_util_attribute_is_git_index_util(): + """Nothing unusual is happening with git.index.util itself.""" + assert git.index.util is sys.modules["git.index.util"] + + +def test_separate_git_util_module_exists(): + """The real git.util and git.index.util modules really are separate. + + The real git.util module can be accessed to import a name ``...` by writing + ``from git.util import ...``, and the module object can be accessed in sys.modules. + """ + assert sys.modules["git.util"] is not sys.modules["git.index.util"] From fc86a238f99a6406ed949a7d690ec0f6be2f31e0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 24 Feb 2024 10:52:49 -0500 Subject: [PATCH 0858/1392] Incompletely change git.index imports to test modattrs.py This also checks if the regression tests in test_imports.py work. This replaces wildcard imports in git.index with explicit imports, and adds git.index.__all__. But this temporarily omits from it the public attributes of git.index that name its Python submodules and are thus are present as an indirect effect of importing names *from* them (since doing so also imports them). This partial change, until it is completed in the next commit, represents the kind of bug that modattrs.py seeks to safeguard against, and verifies that a diff between old and current output of modattrs.py clearly shows it. This diff is temporarily included as ab.diff, which will be deleted in the next commit. The key problem that diff reveals is the changed value of the util attribute of the top-level git module. Due to the way wildcard imports have been used within GitPython for its own modules, expressions like `git.util` (where `git` is the top-level git module) and imports like `from git import util` are actually referring to git.index.util, rather than the util Python submodule of the git module (conceptually git.util), which can only be accessed via `from git.util import ...` or in `sys.modules`. Although this is not an inherently good situation, in practice it has to be maintained at least until the next major version, because reasonable code that uses GitPython would be broken if the situation were changed to be more intuitive. But the more intuitive behavior automatically happens if wildcard imports are replaced with explicit imports of the names they had originally intended to import (in this case, in the top-level git module, which has not yet been done but will be), or if __all__ is added where helpful (in this case, in git.index, which this does). Adding the names explicitly is sufficient to restore the old behavior that is needed for backward compatibility, and has the further advantage that the unintuitive behavior will be explicit once all wildcard imports are replaced and __all__ is added to each module where it would be helpful. The modattrs.py script serves to ensure incompatible changes are not made; this commit checks it. The tests in test_imports.py also cover this specific anticipated incompatibility in git.util, but not all breakages that may arise when refactoring imports and that diff-ing modattrs.py output would help catch. --- ab.diff | 30 ++++++++++++++++++++++++++++++ git/index/__init__.py | 13 +++++++++++-- git/index/base.py | 5 ++--- git/index/typ.py | 5 ++--- 4 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 ab.diff diff --git a/ab.diff b/ab.diff new file mode 100644 index 000000000..34abf9b16 --- /dev/null +++ b/ab.diff @@ -0,0 +1,30 @@ +diff --git a/a b/b +index 81b3f984..7b8c8ede 100644 +--- a/a ++++ b/b +@@ -83,14 +83,12 @@ git: + __path__: ['C:\\Users\\ek\\source\\repos\\GitPython\\git'] + __spec__: ModuleSpec(name='git', loader=<_frozen_importlib_external.SourceFileLoader object at 0x...>, origin='C:\\Users\\ek\\source\\repos\\GitPython\\git\\__init__.py', submodule_search_locations=['C:\\Users\\ek\\source\\repos\\GitPython\\git']) + __version__: 'git' +- base: + cmd: + compat: + config: + db: + diff: + exc: +- fun: + head: + index: + log: +@@ -106,9 +104,8 @@ git: + symbolic: + tag: + to_hex_sha: +- typ: + types: +- util: ++ util: + + + git.cmd: diff --git a/git/index/__init__.py b/git/index/__init__.py index c65722cd8..ba48110fd 100644 --- a/git/index/__init__.py +++ b/git/index/__init__.py @@ -3,5 +3,14 @@ """Initialize the index package.""" -from .base import * # noqa: F401 F403 -from .typ import * # noqa: F401 F403 +__all__ = [ + "BaseIndexEntry", + "BlobFilter", + "CheckoutError", + "IndexEntry", + "IndexFile", + "StageType", +] + +from .base import CheckoutError, IndexFile +from .typ import BaseIndexEntry, BlobFilter, IndexEntry, StageType diff --git a/git/index/base.py b/git/index/base.py index fb91e092c..38c14252b 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -6,6 +6,8 @@ """Module containing :class:`IndexFile`, an Index implementation facilitating all kinds of index manipulations such as querying and merging.""" +__all__ = ("IndexFile", "CheckoutError", "StageType") + import contextlib import datetime import glob @@ -81,9 +83,6 @@ # ------------------------------------------------------------------------------------ -__all__ = ("IndexFile", "CheckoutError", "StageType") - - @contextlib.contextmanager def _named_temporary_file_for_subprocess(directory: PathLike) -> Generator[str, None, None]: """Create a named temporary file git subprocesses can open, deleting it afterward. diff --git a/git/index/typ.py b/git/index/typ.py index ffd76dc46..eb0dd4341 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -3,13 +3,14 @@ """Additional types used by the index.""" +__all__ = ("BlobFilter", "BaseIndexEntry", "IndexEntry", "StageType") + from binascii import b2a_hex from pathlib import Path from .util import pack, unpack from git.objects import Blob - # typing ---------------------------------------------------------------------- from typing import NamedTuple, Sequence, TYPE_CHECKING, Tuple, Union, cast @@ -23,8 +24,6 @@ # --------------------------------------------------------------------------------- -__all__ = ("BlobFilter", "BaseIndexEntry", "IndexEntry", "StageType") - # { Invariants CE_NAMEMASK = 0x0FFF CE_STAGEMASK = 0x3000 From 4badc19f489c177a976d99da85d3c632797f7aeb Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 12:16:33 -0400 Subject: [PATCH 0859/1392] Fix git.index imports - Add the base, fun, typ, and util Python submodules of git.index to git.index.__all__ to restore the old behavior. - Import them explicitly for clarity, even though they are bound to the same-named attributes of git.index either way. This should help human readers know what the entries in __all__ are referring to, and may also help some automated tools. This is a preliminary decision and not necessarily the one that will ultimately be taken in this import refactoring. It *may* cause some kinds of mistakes, such as those arising from ill-advised use of wildcard imports in production code *outside* GitPython, somewhat worse in their effects. - Remove the ab.diff file that showed the problem this solves. This resolves the problem deliberately introduced in the previous incomplete commit, which modattrs.py output and test_imports test results confirm. --- ab.diff | 30 ------------------------------ git/index/__init__.py | 5 +++++ 2 files changed, 5 insertions(+), 30 deletions(-) delete mode 100644 ab.diff diff --git a/ab.diff b/ab.diff deleted file mode 100644 index 34abf9b16..000000000 --- a/ab.diff +++ /dev/null @@ -1,30 +0,0 @@ -diff --git a/a b/b -index 81b3f984..7b8c8ede 100644 ---- a/a -+++ b/b -@@ -83,14 +83,12 @@ git: - __path__: ['C:\\Users\\ek\\source\\repos\\GitPython\\git'] - __spec__: ModuleSpec(name='git', loader=<_frozen_importlib_external.SourceFileLoader object at 0x...>, origin='C:\\Users\\ek\\source\\repos\\GitPython\\git\\__init__.py', submodule_search_locations=['C:\\Users\\ek\\source\\repos\\GitPython\\git']) - __version__: 'git' -- base: - cmd: - compat: - config: - db: - diff: - exc: -- fun: - head: - index: - log: -@@ -106,9 +104,8 @@ git: - symbolic: - tag: - to_hex_sha: -- typ: - types: -- util: -+ util: - - - git.cmd: diff --git a/git/index/__init__.py b/git/index/__init__.py index ba48110fd..2086d67f9 100644 --- a/git/index/__init__.py +++ b/git/index/__init__.py @@ -4,6 +4,10 @@ """Initialize the index package.""" __all__ = [ + "base", + "fun", + "typ", + "util", "BaseIndexEntry", "BlobFilter", "CheckoutError", @@ -12,5 +16,6 @@ "StageType", ] +from . import base, fun, typ, util from .base import CheckoutError, IndexFile from .typ import BaseIndexEntry, BlobFilter, IndexEntry, StageType From 1c9bda222077a6227beed2e06df6f26e2e864807 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 13:59:43 -0400 Subject: [PATCH 0860/1392] Improve relative order of import groups, and __all__, in git.index --- git/index/base.py | 5 +++-- git/index/fun.py | 41 ++++++++++++++++------------------------- git/index/typ.py | 3 ++- git/index/util.py | 7 ++----- 4 files changed, 23 insertions(+), 33 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 38c14252b..b49841435 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -19,6 +19,9 @@ import sys import tempfile +from gitdb.base import IStream +from gitdb.db import MemoryDB + from git.compat import defenc, force_bytes import git.diff as git_diff from git.exc import CheckoutError, GitCommandError, GitError, InvalidGitRepositoryError @@ -33,8 +36,6 @@ unbare_repo, to_bin_sha, ) -from gitdb.base import IStream -from gitdb.db import MemoryDB from .fun import ( S_IFGITLINK, diff --git a/git/index/fun.py b/git/index/fun.py index 001e8f6f2..b24e803a3 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -4,22 +4,28 @@ """Standalone functions to accompany the index implementation and make it more versatile.""" +__all__ = ( + "write_cache", + "read_cache", + "write_tree_from_cache", + "entry_key", + "stat_mode_to_index_mode", + "S_IFGITLINK", + "run_commit_hook", + "hook_path", +) + from io import BytesIO import os import os.path as osp from pathlib import Path -from stat import ( - S_IFDIR, - S_IFLNK, - S_ISLNK, - S_ISDIR, - S_IFMT, - S_IFREG, - S_IXUSR, -) +from stat import S_IFDIR, S_IFLNK, S_IFMT, S_IFREG, S_ISDIR, S_ISLNK, S_IXUSR import subprocess import sys +from gitdb.base import IStream +from gitdb.typ import str_tree_type + from git.cmd import handle_process_output, safer_popen from git.compat import defenc, force_bytes, force_text, safe_decode from git.exc import HookExecutionError, UnmergedEntriesError @@ -29,8 +35,6 @@ tree_to_stream, ) from git.util import IndexFileSHA1Writer, finalize_process -from gitdb.base import IStream -from gitdb.typ import str_tree_type from .typ import BaseIndexEntry, IndexEntry, CE_NAMEMASK, CE_STAGESHIFT from .util import pack, unpack @@ -42,31 +46,18 @@ from git.types import PathLike if TYPE_CHECKING: - from .base import IndexFile from git.db import GitCmdObjectDB from git.objects.tree import TreeCacheTup - # from git.objects.fun import EntryTupOrNone + from .base import IndexFile # ------------------------------------------------------------------------------------ - S_IFGITLINK = S_IFLNK | S_IFDIR """Flags for a submodule.""" CE_NAMEMASK_INV = ~CE_NAMEMASK -__all__ = ( - "write_cache", - "read_cache", - "write_tree_from_cache", - "entry_key", - "stat_mode_to_index_mode", - "S_IFGITLINK", - "run_commit_hook", - "hook_path", -) - def hook_path(name: str, git_dir: PathLike) -> str: """:return: path to the given named hook in the given git repository directory""" diff --git a/git/index/typ.py b/git/index/typ.py index eb0dd4341..0fbcd69f0 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -8,9 +8,10 @@ from binascii import b2a_hex from pathlib import Path -from .util import pack, unpack from git.objects import Blob +from .util import pack, unpack + # typing ---------------------------------------------------------------------- from typing import NamedTuple, Sequence, TYPE_CHECKING, Tuple, Union, cast diff --git a/git/index/util.py b/git/index/util.py index 0bad11571..4aee61bce 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -3,6 +3,8 @@ """Index utilities.""" +__all__ = ("TemporaryFileSwap", "post_clear_cache", "default_index", "git_working_dir") + import contextlib from functools import wraps import os @@ -22,14 +24,9 @@ # --------------------------------------------------------------------------------- - -__all__ = ("TemporaryFileSwap", "post_clear_cache", "default_index", "git_working_dir") - # { Aliases pack = struct.pack unpack = struct.unpack - - # } END aliases From 8b51af36a4943f75fb66d553d55e381859fe3a34 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 14:31:12 -0400 Subject: [PATCH 0861/1392] Improve order of imports and __all__ in git.refs submodules --- git/refs/head.py | 12 ++++++------ git/refs/log.py | 17 ++++++++--------- git/refs/reference.py | 9 +++++---- git/refs/remote.py | 11 +++++------ git/refs/symbolic.py | 11 ++++++----- git/refs/tag.py | 8 +++----- 6 files changed, 33 insertions(+), 35 deletions(-) diff --git a/git/refs/head.py b/git/refs/head.py index 86321d9ea..7e6fc3377 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -6,12 +6,14 @@ Note the distinction between the :class:`HEAD` and :class:`Head` classes. """ +__all__ = ["HEAD", "Head"] + from git.config import GitConfigParser, SectionConstraint -from git.util import join_path from git.exc import GitCommandError +from git.util import join_path -from .symbolic import SymbolicReference from .reference import Reference +from .symbolic import SymbolicReference # typing --------------------------------------------------- @@ -26,8 +28,6 @@ # ------------------------------------------------------------------- -__all__ = ["HEAD", "Head"] - def strip_quotes(string: str) -> str: if string.startswith('"') and string.endswith('"'): @@ -36,8 +36,8 @@ def strip_quotes(string: str) -> str: class HEAD(SymbolicReference): - """Special case of a SymbolicReference representing the repository's HEAD - reference.""" + """Special case of a :class:`~git.refs.symbolic.SymbolicReference` representing the + repository's HEAD reference.""" _HEAD_NAME = "HEAD" _ORIG_HEAD_NAME = "ORIG_HEAD" diff --git a/git/refs/log.py b/git/refs/log.py index f98f56f11..17e3a94b3 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -1,44 +1,43 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ["RefLog", "RefLogEntry"] + from mmap import mmap +import os.path as osp import re import time as _time from git.compat import defenc from git.objects.util import ( - parse_date, Serializable, altz_to_utctz_str, + parse_date, ) from git.util import ( Actor, LockedFD, LockFile, assure_directory_exists, - to_native_path, bin_to_hex, file_contents_ro_filepath, + to_native_path, ) -import os.path as osp - - # typing ------------------------------------------------------------------ -from typing import Iterator, List, Tuple, Union, TYPE_CHECKING +from typing import Iterator, List, Tuple, TYPE_CHECKING, Union from git.types import PathLike if TYPE_CHECKING: from io import BytesIO - from git.refs import SymbolicReference + from git.config import GitConfigParser, SectionConstraint + from git.refs import SymbolicReference # ------------------------------------------------------------------------------ -__all__ = ["RefLog", "RefLogEntry"] - class RefLogEntry(Tuple[str, str, Actor, Tuple[int, int], str]): """Named tuple allowing easy access to the revlog data fields.""" diff --git a/git/refs/reference.py b/git/refs/reference.py index 62fb58420..e5d473779 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -1,7 +1,10 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ["Reference"] + from git.util import IterableObj, LazyMixin + from .symbolic import SymbolicReference, T_References # typing ------------------------------------------------------------------ @@ -15,8 +18,6 @@ # ------------------------------------------------------------------------------ -__all__ = ["Reference"] - # { Utilities @@ -34,7 +35,7 @@ def wrapper(self: T_References, *args: Any) -> _T: return wrapper -# }END utilities +# } END utilities class Reference(SymbolicReference, LazyMixin, IterableObj): @@ -142,7 +143,7 @@ def iter_items( but will return non-detached references as well.""" return cls._iter_items(repo, common_path) - # }END interface + # } END interface # { Remote Interface diff --git a/git/refs/remote.py b/git/refs/remote.py index 3f9c6c6be..b4f4f7b36 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -3,24 +3,23 @@ """Module implementing a remote object allowing easy access to git remotes.""" +__all__ = ["RemoteReference"] + import os from git.util import join_path from .head import Head - -__all__ = ["RemoteReference"] - # typing ------------------------------------------------------------------ -from typing import Any, Iterator, NoReturn, Union, TYPE_CHECKING -from git.types import PathLike +from typing import Any, Iterator, NoReturn, TYPE_CHECKING, Union +from git.types import PathLike if TYPE_CHECKING: + from git.remote import Remote from git.repo import Repo - from git import Remote # ------------------------------------------------------------------------------ diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 2701f9f2b..510850b2e 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -1,10 +1,14 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ["SymbolicReference"] + import os +from gitdb.exc import BadName, BadObject + from git.compat import defenc -from git.objects import Object +from git.objects.base import Object from git.objects.commit import Commit from git.refs.log import RefLog from git.util import ( @@ -15,7 +19,6 @@ join_path_native, to_native_path_linux, ) -from gitdb.exc import BadName, BadObject # typing ------------------------------------------------------------------ @@ -30,6 +33,7 @@ Union, cast, ) + from git.types import AnyGitObject, PathLike if TYPE_CHECKING: @@ -45,9 +49,6 @@ # ------------------------------------------------------------------------------ -__all__ = ["SymbolicReference"] - - def _git_dir(repo: "Repo", path: Union[PathLike, None]) -> PathLike: """Find the git dir that is appropriate for the path.""" name = f"{path}" diff --git a/git/refs/tag.py b/git/refs/tag.py index f653d4e7d..1e38663ae 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -8,10 +8,10 @@ :mod:`git.objects.tag` module. """ -from .reference import Reference - __all__ = ["TagReference", "Tag"] +from .reference import Reference + # typing ------------------------------------------------------------------ from typing import Any, TYPE_CHECKING, Type, Union @@ -19,12 +19,10 @@ from git.types import AnyGitObject, PathLike if TYPE_CHECKING: - from git.objects import Commit - from git.objects import TagObject + from git.objects import Commit, TagObject from git.refs import SymbolicReference from git.repo import Repo - # ------------------------------------------------------------------------------ From b25dd7e6c1b5453fce111a470832e8942430c934 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 14:43:05 -0400 Subject: [PATCH 0862/1392] Replace wildcard imports in git.refs This uses explicit imports rather than wildcard imports in git.refs for names from its submodules. A comment suggested that the import order was deliberate. But each of the modules being imported from defined __all__, and there was no overlap in the names across any of them. The other main reason to import in a specific order is an order dependency in the side effects, but that does not appear to apply, at least not at this time. (In addition, at least for now, this adds explicit imports for the Python submodules of git.refs, so it is clear that they can always be accessed directly in git.refs without further imports, if desired. For clarity, those appear first, and that makes the order of the "from" imports not relevant to such side effects, due to the "from" imports no longer causing modules to be loaded for the first time. However, this is a much less important reason to consider the other imports' reordering okay, because these module imports may end up being removed later during this refactoring; their clarity benefit might not be justified, because if production code outside GitPython ill-advisedly uses wildcard imports, the bad effect of doing so could be increased.) --- git/refs/__init__.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/git/refs/__init__.py b/git/refs/__init__.py index b0233e902..04279901a 100644 --- a/git/refs/__init__.py +++ b/git/refs/__init__.py @@ -1,12 +1,28 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -# Import all modules in order, fix the names they require. +__all__ = [ + "head", + "log", + "reference", + "remote", + "symbolic", + "tag", + "HEAD", + "Head", + "RefLog", + "RefLogEntry", + "Reference", + "RemoteReference", + "SymbolicReference", + "Tag", + "TagReference", +] -from .symbolic import * # noqa: F401 F403 -from .reference import * # noqa: F401 F403 -from .head import * # noqa: F401 F403 -from .tag import * # noqa: F401 F403 -from .remote import * # noqa: F401 F403 - -from .log import * # noqa: F401 F403 +from . import head, log, reference, remote, symbolic, tag +from .head import HEAD, Head +from .log import RefLog, RefLogEntry +from .reference import Reference +from .remote import RemoteReference +from .symbolic import SymbolicReference +from .tag import Tag, TagReference From b32ef6528c0da39b91b6399c5593d10ca41c5865 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 15:01:33 -0400 Subject: [PATCH 0863/1392] Improve order of imports and __all__ in git.repo submodules --- git/repo/base.py | 4 ++-- git/repo/fun.py | 30 ++++++++++++++++-------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index ce164274e..63b21fdbf 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -5,6 +5,8 @@ from __future__ import annotations +__all__ = ("Repo",) + import gc import logging import os @@ -92,8 +94,6 @@ _logger = logging.getLogger(__name__) -__all__ = ("Repo",) - class BlameEntry(NamedTuple): commit: Dict[str, Commit] diff --git a/git/repo/fun.py b/git/repo/fun.py index 738e5816d..1defcb1ac 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -5,18 +5,31 @@ from __future__ import annotations +__all__ = ( + "rev_parse", + "is_git_dir", + "touch", + "find_submodule_git_dir", + "name_to_object", + "short_to_long", + "deref_tag", + "to_commit", + "find_worktree_git_dir", +) + import os import os.path as osp from pathlib import Path import stat from string import digits +from gitdb.exc import BadName, BadObject + from git.cmd import Git from git.exc import WorkTreeRepositoryUnsupported from git.objects import Object from git.refs import SymbolicReference -from git.util import hex_to_bin, bin_to_hex, cygpath -from gitdb.exc import BadName, BadObject +from git.util import cygpath, bin_to_hex, hex_to_bin # Typing ---------------------------------------------------------------------- @@ -29,22 +42,11 @@ from git.objects import Commit, TagObject from git.refs.reference import Reference from git.refs.tag import Tag + from .base import Repo # ---------------------------------------------------------------------------- -__all__ = ( - "rev_parse", - "is_git_dir", - "touch", - "find_submodule_git_dir", - "name_to_object", - "short_to_long", - "deref_tag", - "to_commit", - "find_worktree_git_dir", -) - def touch(filename: str) -> str: with open(filename, "ab"): From 0ba06e9425e90f7f787ac39f48562af54ff41038 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 15:03:49 -0400 Subject: [PATCH 0864/1392] Add git.repo.__all__ and make submodules explicit - No need to import Repo "as Repo". Some tools only recognize this to give the name conceptually public status when it appears in type stub files (.pyi files), and listing it in the newly created __all__ is sufficient to let humans and all tools know it has that status. - As very recently done in git.refs, this explicitly imports the submodules, so it is clear they are available and don't have to be explicitly imported. (Fundamental to the way they are used is that they will end up being imported in order to import Repo.) However, also as in git.refs, it may be that the problems this could cause in some inherently flawed but plausible uses are too greater for it to be justified. So this may be revisited. --- git/repo/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/git/repo/__init__.py b/git/repo/__init__.py index 50a8c6f86..3b784b975 100644 --- a/git/repo/__init__.py +++ b/git/repo/__init__.py @@ -3,4 +3,7 @@ """Initialize the repo package.""" -from .base import Repo as Repo # noqa: F401 +__all__ = ["base", "fun", "Repo"] + +from . import base, fun +from .base import Repo From c946906c4e92512e4294f587eaccaf0029a425df Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 15:29:45 -0400 Subject: [PATCH 0865/1392] Improve order of imports and __all__ in git.objects.* But not yet in git.objects.submodule.* modules. --- git/objects/submodule/base.py | 4 ++-- git/objects/submodule/root.py | 5 +++-- git/objects/submodule/util.py | 24 +++++++++++------------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index d01aa448f..fa60bcdaf 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1,6 +1,8 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ["Submodule", "UpdateProgress"] + import gc from io import BytesIO import logging @@ -68,8 +70,6 @@ # ----------------------------------------------------------------------------- -__all__ = ["Submodule", "UpdateProgress"] - _logger = logging.getLogger(__name__) diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index ae56e5ef4..d93193fa3 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -1,10 +1,13 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ["RootModule", "RootUpdateProgress"] + import logging import git from git.exc import InvalidGitRepositoryError + from .base import Submodule, UpdateProgress from .util import find_first_remote_branch @@ -20,8 +23,6 @@ # ---------------------------------------------------------------------------- -__all__ = ["RootModule", "RootUpdateProgress"] - _logger = logging.getLogger(__name__) diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index 10b994e9b..fda81249b 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -1,12 +1,20 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -import git -from git.exc import InvalidGitRepositoryError -from git.config import GitConfigParser +__all__ = ( + "sm_section", + "sm_name", + "mkhead", + "find_first_remote_branch", + "SubmoduleConfigParser", +) + from io import BytesIO import weakref +import git +from git.config import GitConfigParser +from git.exc import InvalidGitRepositoryError # typing ----------------------------------------------------------------------- @@ -22,15 +30,6 @@ from git import Remote from git.refs import RemoteReference - -__all__ = ( - "sm_section", - "sm_name", - "mkhead", - "find_first_remote_branch", - "SubmoduleConfigParser", -) - # { Utilities @@ -65,7 +64,6 @@ def find_first_remote_branch(remotes: Sequence["Remote"], branch_name: str) -> " # } END utilities - # { Classes From 4e9a2f2facb0da3adb9c6fe5165dcfb5c6627cb7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 15:42:09 -0400 Subject: [PATCH 0866/1392] Improve order of imports and __all__ in git.object.submodule.* --- git/objects/base.py | 12 ++++----- git/objects/blob.py | 6 ++--- git/objects/commit.py | 9 ++++--- git/objects/fun.py | 19 +++++++------- git/objects/submodule/util.py | 9 ++++--- git/objects/tag.py | 14 +++++++--- git/objects/tree.py | 9 +++---- git/objects/util.py | 49 +++++++++++++++++++---------------- 8 files changed, 68 insertions(+), 59 deletions(-) diff --git a/git/objects/base.py b/git/objects/base.py index 22d939aa6..07ff639e5 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -3,15 +3,17 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -import gitdb.typ as dbtyp +__all__ = ("Object", "IndexObject") + import os.path as osp +import gitdb.typ as dbtyp + from git.exc import WorkTreeRepositoryUnsupported -from git.util import LazyMixin, join_path_native, stream_copy, bin_to_hex +from git.util import LazyMixin, bin_to_hex, join_path_native, stream_copy from .util import get_object_type_by_name - # typing ------------------------------------------------------------------ from typing import Any, TYPE_CHECKING, Union @@ -24,16 +26,14 @@ from git.refs.reference import Reference from git.repo import Repo - from .tree import Tree from .blob import Blob from .submodule.base import Submodule + from .tree import Tree IndexObjUnion = Union["Tree", "Blob", "Submodule"] # -------------------------------------------------------------------------- -__all__ = ("Object", "IndexObject") - class Object(LazyMixin): """Base class for classes representing git object types. diff --git a/git/objects/blob.py b/git/objects/blob.py index 122d5f731..50500f550 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -3,17 +3,17 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ("Blob",) + from mimetypes import guess_type import sys -from . import base - if sys.version_info >= (3, 8): from typing import Literal else: from typing_extensions import Literal -__all__ = ("Blob",) +from . import base class Blob(base.IndexObject): diff --git a/git/objects/commit.py b/git/objects/commit.py index 6a60c30bd..b22308726 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -3,6 +3,8 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ("Commit",) + from collections import defaultdict import datetime from io import BytesIO @@ -14,10 +16,12 @@ from time import altzone, daylight, localtime, time, timezone from gitdb import IStream + from git.cmd import Git from git.diff import Diffable -from git.util import hex_to_bin, Actor, Stats, finalize_process +from git.util import Actor, Stats, finalize_process, hex_to_bin +from . import base from .tree import Tree from .util import ( Serializable, @@ -27,7 +31,6 @@ parse_actor_and_date, parse_date, ) -from . import base # typing ------------------------------------------------------------------ @@ -59,8 +62,6 @@ _logger = logging.getLogger(__name__) -__all__ = ("Commit",) - class Commit(base.Object, TraversableIterableObj, Diffable, Serializable): """Wraps a git commit object. diff --git a/git/objects/fun.py b/git/objects/fun.py index 5bd8a3d62..9bd2d8f10 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -3,8 +3,14 @@ """Functions that are supposed to be as fast as possible.""" -from stat import S_ISDIR +__all__ = ( + "tree_to_stream", + "tree_entries_from_data", + "traverse_trees_recursive", + "traverse_tree_recursive", +) +from stat import S_ISDIR from git.compat import safe_decode, defenc @@ -23,22 +29,15 @@ if TYPE_CHECKING: from _typeshed import ReadableBuffer + from git import GitCmdObjectDB -EntryTup = Tuple[bytes, int, str] # same as TreeCacheTup in tree.py +EntryTup = Tuple[bytes, int, str] # Same as TreeCacheTup in tree.py. EntryTupOrNone = Union[EntryTup, None] # --------------------------------------------------- -__all__ = ( - "tree_to_stream", - "tree_entries_from_data", - "traverse_trees_recursive", - "traverse_tree_recursive", -) - - def tree_to_stream(entries: Sequence[EntryTup], write: Callable[["ReadableBuffer"], Union[int, None]]) -> None: """Write the given list of entries into a stream using its ``write`` method. diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index fda81249b..3a09d6a17 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -23,12 +23,13 @@ from git.types import PathLike if TYPE_CHECKING: - from .base import Submodule from weakref import ReferenceType + + from git.refs import Head, RemoteReference + from git.remote import Remote from git.repo import Repo - from git.refs import Head - from git import Remote - from git.refs import RemoteReference + + from .base import Submodule # { Utilities diff --git a/git/objects/tag.py b/git/objects/tag.py index 52d79751f..5ad311590 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -9,12 +9,17 @@ For lightweight tags, see the :mod:`git.refs.tag` module. """ +__all__ = ("TagObject",) + import sys +from git.compat import defenc +from git.util import hex_to_bin + from . import base from .util import get_object_type_by_name, parse_actor_and_date -from ..util import hex_to_bin -from ..compat import defenc + +# typing ---------------------------------------------- from typing import List, TYPE_CHECKING, Union @@ -26,11 +31,12 @@ if TYPE_CHECKING: from git.repo import Repo from git.util import Actor - from .commit import Commit + from .blob import Blob + from .commit import Commit from .tree import Tree -__all__ = ("TagObject",) +# --------------------------------------------------- class TagObject(base.Object): diff --git a/git/objects/tree.py b/git/objects/tree.py index c74df58a9..18ccf2f30 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -3,16 +3,18 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ("TreeModifier", "Tree") + import sys import git.diff as git_diff from git.util import IterableList, join_path, to_bin_sha +from . import util from .base import IndexObjUnion, IndexObject from .blob import Blob from .fun import tree_entries_from_data, tree_to_stream from .submodule.base import Submodule -from . import util # typing ------------------------------------------------- @@ -39,23 +41,20 @@ if TYPE_CHECKING: from io import BytesIO + from git.repo import Repo TreeCacheTup = Tuple[bytes, int, str] TraversedTreeTup = Union[Tuple[Union["Tree", None], IndexObjUnion, Tuple["Submodule", "Submodule"]]] - # def is_tree_cache(inp: Tuple[bytes, int, str]) -> TypeGuard[TreeCacheTup]: # return isinstance(inp[0], bytes) and isinstance(inp[1], int) and isinstance([inp], str) # -------------------------------------------------------- - cmp: Callable[[str, str], int] = lambda a, b: (a > b) - (a < b) -__all__ = ("TreeModifier", "Tree") - class TreeModifier: """A utility class providing methods to alter the underlying cache in a list-like diff --git a/git/objects/util.py b/git/objects/util.py index 297b33b70..6f6927b47 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -5,26 +5,40 @@ """General utility functions.""" +__all__ = ( + "get_object_type_by_name", + "parse_date", + "parse_actor_and_date", + "ProcessStreamAdapter", + "Traversable", + "altz_to_utctz_str", + "utctz_to_altz", + "verify_utctz", + "Actor", + "tzoffset", + "utc", +) + from abc import ABC, abstractmethod import calendar from collections import deque from datetime import datetime, timedelta, tzinfo -from string import digits import re +from string import digits import time import warnings -from git.util import IterableList, IterableObj, Actor +from git.util import Actor, IterableList, IterableObj # typing ------------------------------------------------------------ + from typing import ( Any, Callable, Deque, - Iterator, # Generic, + Iterator, NamedTuple, - overload, Sequence, TYPE_CHECKING, Tuple, @@ -32,19 +46,22 @@ TypeVar, Union, cast, + overload, ) from git.types import Has_id_attribute, Literal # , _T if TYPE_CHECKING: from io import BytesIO, StringIO - from .commit import Commit - from .blob import Blob - from .tag import TagObject - from .tree import Tree, TraversedTreeTup from subprocess import Popen - from .submodule.base import Submodule + from git.types import Protocol, runtime_checkable + + from .blob import Blob + from .commit import Commit + from .submodule.base import Submodule + from .tag import TagObject + from .tree import TraversedTreeTup, Tree else: # Protocol = Generic[_T] # Needed for typing bug #572? Protocol = ABC @@ -68,20 +85,6 @@ class TraverseNT(NamedTuple): # -------------------------------------------------------------------- -__all__ = ( - "get_object_type_by_name", - "parse_date", - "parse_actor_and_date", - "ProcessStreamAdapter", - "Traversable", - "altz_to_utctz_str", - "utctz_to_altz", - "verify_utctz", - "Actor", - "tzoffset", - "utc", -) - ZERO = timedelta(0) # { Functions From c58be4c0b6e82dc1c0445f9e41e53263f5d2ec4e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 15:44:52 -0400 Subject: [PATCH 0867/1392] Remove a bit of old commented-out code in git.objects.* --- git/objects/tree.py | 4 ---- git/objects/util.py | 4 +--- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/git/objects/tree.py b/git/objects/tree.py index 18ccf2f30..d8320b319 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -48,9 +48,6 @@ TraversedTreeTup = Union[Tuple[Union["Tree", None], IndexObjUnion, Tuple["Submodule", "Submodule"]]] -# def is_tree_cache(inp: Tuple[bytes, int, str]) -> TypeGuard[TreeCacheTup]: -# return isinstance(inp[0], bytes) and isinstance(inp[1], int) and isinstance([inp], str) - # -------------------------------------------------------- cmp: Callable[[str, str], int] = lambda a, b: (a > b) - (a < b) @@ -124,7 +121,6 @@ def add(self, sha: bytes, mode: int, name: str, force: bool = False) -> "TreeMod index = self._index_by_name(name) item = (sha, mode, name) - # assert is_tree_cache(item) if index == -1: self._cache.append(item) diff --git a/git/objects/util.py b/git/objects/util.py index 6f6927b47..9491b067c 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -36,7 +36,6 @@ Any, Callable, Deque, - # Generic, Iterator, NamedTuple, Sequence, @@ -49,7 +48,7 @@ overload, ) -from git.types import Has_id_attribute, Literal # , _T +from git.types import Has_id_attribute, Literal if TYPE_CHECKING: from io import BytesIO, StringIO @@ -63,7 +62,6 @@ from .tag import TagObject from .tree import TraversedTreeTup, Tree else: - # Protocol = Generic[_T] # Needed for typing bug #572? Protocol = ABC def runtime_checkable(f): From 01c95ebcc7d41bc68e1f64f9b1ec01c30f29d591 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 16:38:39 -0400 Subject: [PATCH 0868/1392] Don't patch IndexObject and Object into git.objects.submodule.util Such patching, which was introduced in 9519f18, seems no longer to be necessary. Since git.objects.submodule.util.__all__ has existed for a long time without including those names, they are not conceptually public attributes of git.objects.submodule.util, so they should not be in use by any code outside GitPython either. The modattrs.py script shows the change, as expected, showing these two names as no longer being in the git.objects.submodule.util module dictionary, in output of: python modattrs.py >b git diff --no-index a b However, because the removal is intentional, this is okay. --- git/objects/__init__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/git/objects/__init__.py b/git/objects/__init__.py index 1061ec874..d636873a0 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -8,17 +8,10 @@ from .base import * # noqa: F403 from .blob import * # noqa: F403 from .commit import * # noqa: F403 -from .submodule import util as smutil from .submodule.base import * # noqa: F403 from .submodule.root import * # noqa: F403 from .tag import * # noqa: F403 from .tree import * # noqa: F403 -# Fix import dependency - add IndexObject to the util module, so that it can be imported -# by the submodule.base. -smutil.IndexObject = IndexObject # type: ignore[attr-defined] # noqa: F405 -smutil.Object = Object # type: ignore[attr-defined] # noqa: F405 -del smutil - # Must come after submodule was made available. __all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] From f89d065742ba5b62056f1aa679e1197124bf7bcf Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 18:50:37 -0400 Subject: [PATCH 0869/1392] Fix git.objects.__all__ and make submodules explicit The submodules being made explicit here are of course Python submodules, not git submodules. The git.objects.submodule Python submodule (which is *about* git submodules) is made explicit here (as are the names imported from that Python submodule's own Python submodules) along with the other Python submodules of git.objects. Unlike some other submodules, but like the top-level git module until #1659, git.objects already defined __all__ but it was dynamically constructed. As with git.__all__ previously (as noted in #1859), I have used https://github.com/EliahKagan/deltall here to check that it is safe, sufficient, and probably necessary to replace this dynamically constructed __all__ with a literal list of strings in which all of the original elements are included. See: https://gist.github.com/EliahKagan/e627603717ca5f9cafaf8de9d9f27ad7 Running the modattrs.py script, and diffing against the output from before the current import refactoring began, reveals two changes to module contents as a result of the change here: - git.objects no longer contains `inspect`, which it imported just to build the dynamically constructed __all__. Because this was not itself included in that __all__ or otherwise made public or used out of git.objects, that is okay. This is exactly analogous to the situation in 8197e90, which removed it from the top-level git module after #1659. - The top-level git module now has has new attributes blob, commit, submodule, and tree, aliasing the corresponding modules. This has happened as a result of them being put in git.objects.__all__ along with various names imported from them. (As noted in some prior commits, there are tradeoffs associated with doing this, and it may be that such elements of __all__ should be removed, here and elsewhere.) --- git/objects/__init__.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/git/objects/__init__.py b/git/objects/__init__.py index d636873a0..692a468d6 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -3,15 +3,30 @@ """Import all submodules' main classes into the package space.""" -import inspect +__all__ = [ + "base", + "blob", + "commit", + "submodule", + "tag", + "tree", + "IndexObject", + "Object", + "Blob", + "Commit", + "Submodule", + "UpdateProgress", + "RootModule", + "RootUpdateProgress", + "TagObject", + "Tree", + "TreeModifier", +] -from .base import * # noqa: F403 -from .blob import * # noqa: F403 -from .commit import * # noqa: F403 -from .submodule.base import * # noqa: F403 -from .submodule.root import * # noqa: F403 -from .tag import * # noqa: F403 -from .tree import * # noqa: F403 - -# Must come after submodule was made available. -__all__ = [name for name, obj in locals().items() if not (name.startswith("_") or inspect.ismodule(obj))] +from .base import IndexObject, Object +from .blob import Blob +from .commit import Commit +from .submodule.base import Submodule, UpdateProgress +from .submodule.root import RootModule, RootUpdateProgress +from .tag import TagObject +from .tree import Tree, TreeModifier From 3786307f965d38cde9e251d191f71ff26903935d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 19:10:17 -0400 Subject: [PATCH 0870/1392] Make git.objects.util module docstring more specific So git.objects.util is less likely to be confused with git.util. (The modattrs.py script reveals the change in the value of git.objects.util.__doc__, as expected.) --- git/objects/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/util.py b/git/objects/util.py index 9491b067c..08aef132a 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -3,7 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""General utility functions.""" +"""General utility functions for working with git objects.""" __all__ = ( "get_object_type_by_name", From de540b78caa71c9f1a0f098c0d1608287c8927ac Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 19:17:48 -0400 Subject: [PATCH 0871/1392] Add __all__ and imports in git.objects.submodule This is the top-level of the git.objects.submodule subpackage, for its own Python submodules. This appears only not to have been done before because of a cyclic import problem involving imports that are no longer present. Doing it improves consistency, since all the other subpackages under the top-level git package already do this. The modattrs.py script reveals the four new attributes of git.objects.submodule for the four classes obtained by the new imports, as expected. (The explicit module imports do not change the attribues because they are the same attributes as come into existence due to those Python submodules being imported anywhere in any style.) --- git/objects/__init__.py | 3 +-- git/objects/submodule/__init__.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/git/objects/__init__.py b/git/objects/__init__.py index 692a468d6..4f18c8754 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -26,7 +26,6 @@ from .base import IndexObject, Object from .blob import Blob from .commit import Commit -from .submodule.base import Submodule, UpdateProgress -from .submodule.root import RootModule, RootUpdateProgress +from .submodule import RootModule, RootUpdateProgress, Submodule, UpdateProgress from .tag import TagObject from .tree import Tree, TreeModifier diff --git a/git/objects/submodule/__init__.py b/git/objects/submodule/__init__.py index b11b568f2..383439545 100644 --- a/git/objects/submodule/__init__.py +++ b/git/objects/submodule/__init__.py @@ -1,5 +1,16 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -# NOTE: Cannot import anything here as the top-level __init__ has to handle -# our dependencies. +__all__ = [ + "base", + "root", + "util", + "Submodule", + "UpdateProgress", + "RootModule", + "RootUpdateProgress", +] + +from . import base, root, util +from .base import Submodule, UpdateProgress +from .root import RootModule, RootUpdateProgress From a05597a1aeacec0c18772944208ab2757b0db34b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 19:48:07 -0400 Subject: [PATCH 0872/1392] Improve how imports and __all__ are written in git.util + Update the detailed comment about the unused import situation. --- git/util.py | 102 +++++++++++++++++++++++++++------------------------- 1 file changed, 53 insertions(+), 49 deletions(-) diff --git a/git/util.py b/git/util.py index 37986edaa..cf5949693 100644 --- a/git/util.py +++ b/git/util.py @@ -3,6 +3,32 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import sys + +__all__ = [ + "stream_copy", + "join_path", + "to_native_path_linux", + "join_path_native", + "Stats", + "IndexFileSHA1Writer", + "IterableObj", + "IterableList", + "BlockingLockFile", + "LockFile", + "Actor", + "get_user_id", + "assure_directory_exists", + "RemoteProgress", + "CallableRemoteProgress", + "rmtree", + "unbare_repo", + "HIDE_WINDOWS_KNOWN_ERRORS", +] + +if sys.platform == "win32": + __all__.append("to_native_path_windows") + from abc import abstractmethod import contextlib from functools import wraps @@ -16,11 +42,27 @@ import shutil import stat import subprocess -import sys import time from urllib.parse import urlsplit, urlunsplit import warnings +# NOTE: Unused imports can be improved now that CI testing has fully resumed. Some of +# these be used indirectly through other GitPython modules, which avoids having to write +# gitdb all the time in their imports. They are not in __all__, at least currently, +# because they could be removed or changed at any time, and so should not be considered +# conceptually public to code outside GitPython. Linters of course do not like it. +from gitdb.util import ( # noqa: F401 # @IgnorePep8 + LazyMixin, # @UnusedImport + LockedFD, # @UnusedImport + bin_to_hex, # @UnusedImport + file_contents_ro_filepath, # @UnusedImport + file_contents_ro, # @UnusedImport + hex_to_bin, # @UnusedImport + make_sha, + to_bin_sha, # @UnusedImport + to_hex_sha, # @UnusedImport +) + # typing --------------------------------------------------------- from typing import ( @@ -37,73 +79,36 @@ Pattern, Sequence, Tuple, + TYPE_CHECKING, TypeVar, Union, - TYPE_CHECKING, cast, overload, ) if TYPE_CHECKING: + from git.cmd import Git + from git.config import GitConfigParser, SectionConstraint from git.remote import Remote from git.repo.base import Repo - from git.config import GitConfigParser, SectionConstraint - from git import Git -from .types import ( +from git.types import ( + Files_TD, + Has_id_attribute, + HSH_TD, Literal, - SupportsIndex, - Protocol, - runtime_checkable, # because behind py version guards PathLike, - HSH_TD, + Protocol, + SupportsIndex, Total_TD, - Files_TD, # aliases - Has_id_attribute, + runtime_checkable, ) # --------------------------------------------------------------------- -from gitdb.util import ( # noqa: F401 # @IgnorePep8 - make_sha, - LockedFD, # @UnusedImport - file_contents_ro, # @UnusedImport - file_contents_ro_filepath, # @UnusedImport - LazyMixin, # @UnusedImport - to_hex_sha, # @UnusedImport - to_bin_sha, # @UnusedImport - bin_to_hex, # @UnusedImport - hex_to_bin, # @UnusedImport -) - T_IterableObj = TypeVar("T_IterableObj", bound=Union["IterableObj", "Has_id_attribute"], covariant=True) # So IterableList[Head] is subtype of IterableList[IterableObj]. -# NOTE: Some of the unused imports might be used/imported by others. -# Handle once test-cases are back up and running. -# Most of these are unused here, but are for use by git-python modules so these -# don't see gitdb all the time. Flake of course doesn't like it. -__all__ = [ - "stream_copy", - "join_path", - "to_native_path_linux", - "join_path_native", - "Stats", - "IndexFileSHA1Writer", - "IterableObj", - "IterableList", - "BlockingLockFile", - "LockFile", - "Actor", - "get_user_id", - "assure_directory_exists", - "RemoteProgress", - "CallableRemoteProgress", - "rmtree", - "unbare_repo", - "HIDE_WINDOWS_KNOWN_ERRORS", -] - _logger = logging.getLogger(__name__) @@ -292,7 +297,6 @@ def to_native_path_linux(path: PathLike) -> str: path = str(path) return path.replace("\\", "/") - __all__.append("to_native_path_windows") to_native_path = to_native_path_windows else: # No need for any work on Linux. From 2053a3d66c1667f3b130347e9984779f94875719 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 19:58:34 -0400 Subject: [PATCH 0873/1392] Remove old commented-out change_type assertions in git.diff --- git/diff.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/git/diff.py b/git/diff.py index a6322ff57..54a5cf9af 100644 --- a/git/diff.py +++ b/git/diff.py @@ -40,11 +40,6 @@ Lit_change_type = Literal["A", "D", "C", "M", "R", "T", "U"] - -# def is_change_type(inp: str) -> TypeGuard[Lit_change_type]: -# # return True -# return inp in ['A', 'D', 'C', 'M', 'R', 'T', 'U'] - # ------------------------------------------------------------------------ @@ -693,7 +688,6 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non # Change type can be R100 # R: status letter # 100: score (in case of copy and rename) - # assert is_change_type(_change_type[0]), f"Unexpected value for change_type received: {_change_type[0]}" change_type: Lit_change_type = cast(Lit_change_type, _change_type[0]) score_str = "".join(_change_type[1:]) score = int(score_str) if score_str.isdigit() else None From b8bab43db59c8a110d11dd16e24d61adf65c18fd Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 20:02:55 -0400 Subject: [PATCH 0874/1392] Remove old commented-out flagKeyLiteral assertions in git.remote --- git/remote.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/git/remote.py b/git/remote.py index 2c452022e..65bafc0e9 100644 --- a/git/remote.py +++ b/git/remote.py @@ -50,10 +50,6 @@ flagKeyLiteral = Literal[" ", "!", "+", "-", "*", "=", "t", "?"] -# def is_flagKeyLiteral(inp: str) -> TypeGuard[flagKeyLiteral]: -# return inp in [' ', '!', '+', '-', '=', '*', 't', '?'] - - # ------------------------------------------------------------- _logger = logging.getLogger(__name__) @@ -415,7 +411,6 @@ def _from_line(cls, repo: "Repo", line: str, fetch_line: str) -> "FetchInfo": remote_local_ref_str, note, ) = match.groups() - # assert is_flagKeyLiteral(control_character), f"{control_character}" control_character = cast(flagKeyLiteral, control_character) try: _new_hex_sha, _fetch_operation, fetch_note = fetch_line.split("\t") From 3d4e47623ef7dc76883abfd5d44fcaef92e7bd1c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 20:07:17 -0400 Subject: [PATCH 0875/1392] Improve how second-level imports and __all__ are written These are in the modules that are directly under the top-level git module (and are not themselves subpackages, with their own submodules in the Python sense), except for: - git.util, where this change was very recently made. - git.compat, where no improvements of this kind were needed. - git.types, which currently has no __all__ and will only benefit from it if what should go in it is carefully considered (and where the imports themselves are grouped, sorted, and formatted already). --- git/cmd.py | 15 +++++++-------- git/config.py | 7 ++++--- git/db.py | 13 ++++++------- git/diff.py | 24 +++++++++++------------- git/exc.py | 4 +++- git/remote.py | 4 ++-- 6 files changed, 33 insertions(+), 34 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 2862b1600..7459fae97 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -5,18 +5,20 @@ from __future__ import annotations -import re +__all__ = ("Git",) + import contextlib import io import itertools import logging import os +import re import signal -from subprocess import Popen, PIPE, DEVNULL import subprocess +from subprocess import DEVNULL, PIPE, Popen import sys -import threading from textwrap import dedent +import threading from git.compat import defenc, force_bytes, safe_decode from git.exc import ( @@ -57,12 +59,11 @@ overload, ) -from git.types import PathLike, Literal, TBD +from git.types import Literal, PathLike, TBD if TYPE_CHECKING: - from git.repo.base import Repo from git.diff import DiffIndex - + from git.repo.base import Repo # --------------------------------------------------------------------------------- @@ -84,8 +85,6 @@ _logger = logging.getLogger(__name__) -__all__ = ("Git",) - # ============================================================================== ## @name Utilities diff --git a/git/config.py b/git/config.py index f74d290cc..1c9761c1b 100644 --- a/git/config.py +++ b/git/config.py @@ -5,6 +5,8 @@ """Parser for reading and writing configuration files.""" +__all__ = ("GitConfigParser", "SectionConstraint") + import abc import configparser as cp import fnmatch @@ -40,9 +42,10 @@ from git.types import Lit_config_levels, ConfigLevels_Tup, PathLike, assert_never, _T if TYPE_CHECKING: - from git.repo.base import Repo from io import BytesIO + from git.repo.base import Repo + T_ConfigParser = TypeVar("T_ConfigParser", bound="GitConfigParser") T_OMD_value = TypeVar("T_OMD_value", str, bytes, int, float, bool) @@ -58,8 +61,6 @@ # ------------------------------------------------------------- -__all__ = ("GitConfigParser", "SectionConstraint") - _logger = logging.getLogger(__name__) CONFIG_LEVELS: ConfigLevels_Tup = ("system", "user", "global", "repository") diff --git a/git/db.py b/git/db.py index 5b2ca4de2..302192bdf 100644 --- a/git/db.py +++ b/git/db.py @@ -3,27 +3,26 @@ """Module with our own gitdb implementation - it uses the git command.""" -from git.util import bin_to_hex, hex_to_bin -from gitdb.base import OInfo, OStream -from gitdb.db import GitDB -from gitdb.db import LooseObjectDB +__all__ = ("GitCmdObjectDB", "GitDB") +from gitdb.base import OInfo, OStream +from gitdb.db import GitDB, LooseObjectDB from gitdb.exc import BadObject + +from git.util import bin_to_hex, hex_to_bin from git.exc import GitCommandError # typing------------------------------------------------- from typing import TYPE_CHECKING + from git.types import PathLike if TYPE_CHECKING: from git.cmd import Git - # -------------------------------------------------------- -__all__ = ("GitCmdObjectDB", "GitDB") - class GitCmdObjectDB(LooseObjectDB): """A database representing the default git object store, which includes loose diff --git a/git/diff.py b/git/diff.py index 54a5cf9af..1381deca0 100644 --- a/git/diff.py +++ b/git/diff.py @@ -3,17 +3,17 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = ("DiffConstants", "NULL_TREE", "INDEX", "Diffable", "DiffIndex", "Diff") + import enum import re from git.cmd import handle_process_output from git.compat import defenc +from git.objects.blob import Blob +from git.objects.util import mode_str_to_int from git.util import finalize_process, hex_to_bin -from .objects.blob import Blob -from .objects.util import mode_str_to_int - - # typing ------------------------------------------------------------------ from typing import ( @@ -23,29 +23,27 @@ Match, Optional, Tuple, + TYPE_CHECKING, TypeVar, Union, - TYPE_CHECKING, cast, ) from git.types import Literal, PathLike if TYPE_CHECKING: - from .objects.tree import Tree - from .objects import Commit - from git.repo.base import Repo - from git.objects.base import IndexObject from subprocess import Popen - from git import Git + + from git.cmd import Git + from git.objects.base import IndexObject + from git.objects.commit import Commit + from git.objects.tree import Tree + from git.repo.base import Repo Lit_change_type = Literal["A", "D", "C", "M", "R", "T", "U"] # ------------------------------------------------------------------------ -__all__ = ("DiffConstants", "NULL_TREE", "INDEX", "Diffable", "DiffIndex", "Diff") - - @enum.unique class DiffConstants(enum.Enum): """Special objects for :meth:`Diffable.diff`. diff --git a/git/exc.py b/git/exc.py index 9f6462b39..583eee8c1 100644 --- a/git/exc.py +++ b/git/exc.py @@ -42,12 +42,14 @@ ParseError, UnsupportedOperation, ) + from git.compat import safe_decode from git.util import remove_password_if_present # typing ---------------------------------------------------- -from typing import List, Sequence, Tuple, Union, TYPE_CHECKING +from typing import List, Sequence, Tuple, TYPE_CHECKING, Union + from git.types import PathLike if TYPE_CHECKING: diff --git a/git/remote.py b/git/remote.py index 65bafc0e9..54a5c459c 100644 --- a/git/remote.py +++ b/git/remote.py @@ -5,6 +5,8 @@ """Module implementing a remote object allowing easy access to git remotes.""" +__all__ = ("RemoteProgress", "PushInfo", "FetchInfo", "Remote") + import contextlib import logging import re @@ -54,8 +56,6 @@ _logger = logging.getLogger(__name__) -__all__ = ("RemoteProgress", "PushInfo", "FetchInfo", "Remote") - # { Utilities From 6318eea9743efec557bd7dbfdea46ddc21f654f2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 20:27:20 -0400 Subject: [PATCH 0876/1392] Make F401 "unused import" suppressions more specific In git.compat and git.util. This applies them individually to each name that is known to be an unused import, rather than to entire import statements (except where the entire statement fit on one line and everything it imported is known to be unused). Either Ruff has the ability to accept this more granular style of suppression, or I had been unaware of it from flake8 (used before). I have veriifed that the suppressions are not superfluous: with no suppressions, Ruff does warn. This commit makes no change in git.types because it looks like no suppression at all may be needed there anymore; that will be covered in the next commit. This also removes the old `@UnusedImport` comments, which had been kept before because they were more granular; this specificity is now achieved by comments the tools being used can recognize. --- git/compat.py | 14 +++++++------- git/util.py | 18 +++++++++--------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/git/compat.py b/git/compat.py index 6f5376c9d..4ede8c985 100644 --- a/git/compat.py +++ b/git/compat.py @@ -14,18 +14,18 @@ import os import sys -from gitdb.utils.encoding import force_bytes, force_text # noqa: F401 # @UnusedImport +from gitdb.utils.encoding import force_bytes, force_text # noqa: F401 # typing -------------------------------------------------------------------- -from typing import ( # noqa: F401 - Any, +from typing import ( + Any, # noqa: F401 AnyStr, - Dict, - IO, + Dict, # noqa: F401 + IO, # noqa: F401 Optional, - Tuple, - Type, + Tuple, # noqa: F401 + Type, # noqa: F401 Union, overload, ) diff --git a/git/util.py b/git/util.py index cf5949693..a56b63d69 100644 --- a/git/util.py +++ b/git/util.py @@ -51,16 +51,16 @@ # gitdb all the time in their imports. They are not in __all__, at least currently, # because they could be removed or changed at any time, and so should not be considered # conceptually public to code outside GitPython. Linters of course do not like it. -from gitdb.util import ( # noqa: F401 # @IgnorePep8 - LazyMixin, # @UnusedImport - LockedFD, # @UnusedImport - bin_to_hex, # @UnusedImport - file_contents_ro_filepath, # @UnusedImport - file_contents_ro, # @UnusedImport - hex_to_bin, # @UnusedImport +from gitdb.util import ( + LazyMixin, # noqa: F401 + LockedFD, # noqa: F401 + bin_to_hex, # noqa: F401 + file_contents_ro_filepath, # noqa: F401 + file_contents_ro, # noqa: F401 + hex_to_bin, # noqa: F401 make_sha, - to_bin_sha, # @UnusedImport - to_hex_sha, # @UnusedImport + to_bin_sha, # noqa: F401 + to_hex_sha, # noqa: F401 ) # typing --------------------------------------------------------- From 31bc8a4e062841e041e9fff664bb08366d8fce48 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 20:33:33 -0400 Subject: [PATCH 0877/1392] Remove unneeded F401 "Unused import" suppressions In git.types. It appears Ruff, unlike flake8, recognizes "import X as X" to mean that "X" should not be considered unused, even when it appears outside a .pyi type stub file where that notation is more commonly used. Those imports in git.types may benefit from being changed in some way that uses a syntax whose intent is clearer in context, and depending on how that is done it may even be necessary to bring back suppressions. If so, they can be written more specifically. (For now, changing them would express more than is known about what names that are only present in git.types becuase it imports them should continue to be imported, should be considered conceptually public, or should be condered suitable for use within GitPython.) --- git/types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/types.py b/git/types.py index e3ae9e3d5..230422dff 100644 --- a/git/types.py +++ b/git/types.py @@ -3,7 +3,7 @@ import os import sys -from typing import ( # noqa: F401 +from typing import ( Any, Callable, Dict, @@ -17,7 +17,7 @@ ) if sys.version_info >= (3, 8): - from typing import ( # noqa: F401 + from typing import ( Literal, Protocol, SupportsIndex as SupportsIndex, @@ -25,7 +25,7 @@ runtime_checkable, ) else: - from typing_extensions import ( # noqa: F401 + from typing_extensions import ( Literal, Protocol, SupportsIndex as SupportsIndex, From abbe74d030a647664f646a8d43e071608593b6a3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 20:40:13 -0400 Subject: [PATCH 0878/1392] Fix a tiny import sorting nit --- git/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index a56b63d69..5839f9720 100644 --- a/git/util.py +++ b/git/util.py @@ -55,8 +55,8 @@ LazyMixin, # noqa: F401 LockedFD, # noqa: F401 bin_to_hex, # noqa: F401 - file_contents_ro_filepath, # noqa: F401 file_contents_ro, # noqa: F401 + file_contents_ro_filepath, # noqa: F401 hex_to_bin, # noqa: F401 make_sha, to_bin_sha, # noqa: F401 From 774525087bcc0389bf585b88ee38eabf69933183 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 22:09:52 -0400 Subject: [PATCH 0879/1392] Replace wildcard imports in top-level git module - Use explicit imports instead of * imports. - Remove now-unneeded linter suppressions. - Alphabetize inside the try-block, though this will be undone. This currently fails to import due to a cyclic import error, so the third change, alphabetizing the imports, will have to be undone (at least in the absence of other changes). It is not merely that they should not be reordered in a way that makes them cross into or out of the try-block, but that within the try block not all orders will work. There will be more to do for backward compatibility, but the modattrs.py script, which imports the top-level git module, cannot be run until the cyclic import problem is fixed. --- git/__init__.py | 95 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 17 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index ca5bed7a3..ebcaf89a2 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -5,7 +5,7 @@ # @PydevCodeAnalysisIgnore -__all__ = [ # noqa: F405 +__all__ = [ "Actor", "AmbiguousObjectName", "BadName", @@ -88,32 +88,93 @@ __version__ = "git" -from typing import List, Optional, Sequence, Tuple, Union, TYPE_CHECKING +from typing import List, Optional, Sequence, TYPE_CHECKING, Tuple, Union from gitdb.util import to_hex_sha -from git.exc import * # noqa: F403 # @NoMove @IgnorePep8 + +from git.exc import ( + AmbiguousObjectName, + BadName, + BadObject, + BadObjectType, + CacheError, + CheckoutError, + CommandError, + GitCommandError, + GitCommandNotFound, + GitError, + HookExecutionError, + InvalidDBRoot, + InvalidGitRepositoryError, + NoSuchPathError, + ODBError, + ParseError, + RepositoryDirtyError, + UnmergedEntriesError, + UnsafeOptionError, + UnsafeProtocolError, + UnsupportedOperation, + WorkTreeRepositoryUnsupported, +) from git.types import PathLike try: - from git.compat import safe_decode # @NoMove @IgnorePep8 - from git.config import GitConfigParser # @NoMove @IgnorePep8 - from git.objects import * # noqa: F403 # @NoMove @IgnorePep8 - from git.refs import * # noqa: F403 # @NoMove @IgnorePep8 - from git.diff import * # noqa: F403 # @NoMove @IgnorePep8 - from git.db import * # noqa: F403 # @NoMove @IgnorePep8 - from git.cmd import Git # @NoMove @IgnorePep8 - from git.repo import Repo # @NoMove @IgnorePep8 - from git.remote import * # noqa: F403 # @NoMove @IgnorePep8 - from git.index import * # noqa: F403 # @NoMove @IgnorePep8 - from git.util import ( # @NoMove @IgnorePep8 - LockFile, + from git.cmd import Git # @NoMove + from git.compat import safe_decode # @NoMove + from git.config import GitConfigParser # @NoMove + from git.db import GitCmdObjectDB, GitDB # @NoMove + from git.diff import ( # @NoMove + INDEX, + NULL_TREE, + Diff, + DiffConstants, + DiffIndex, + Diffable, + ) + from git.index import ( # @NoMove + BaseIndexEntry, + BlobFilter, + CheckoutError, + IndexEntry, + IndexFile, + StageType, + util, # noqa: F401 # For backward compatibility. + ) + from git.objects import ( # @NoMove + Blob, + Commit, + IndexObject, + Object, + RootModule, + RootUpdateProgress, + Submodule, + TagObject, + Tree, + TreeModifier, + UpdateProgress, + ) + from git.refs import ( # @NoMove + HEAD, + Head, + RefLog, + RefLogEntry, + Reference, + RemoteReference, + SymbolicReference, + Tag, + TagReference, + ) + from git.remote import FetchInfo, PushInfo, Remote, RemoteProgress # @NoMove + from git.repo import Repo # @NoMove + from git.util import ( # @NoMove + Actor, BlockingLockFile, + LockFile, Stats, - Actor, remove_password_if_present, rmtree, ) -except GitError as _exc: # noqa: F405 +except GitError as _exc: raise ImportError("%s: %s" % (_exc.__class__.__name__, _exc)) from _exc # { Initialize git executable path From 64c9efdad216954b19cb91a4c04ea2de9d598159 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 22:13:45 -0400 Subject: [PATCH 0880/1392] Restore relative order to fix circular import error This still uses all explicit rather than wildcard imports (and still omits suppressions that are no longer needed due to wildcard imports not being used). But it brings back the old relative order of the `from ... import ...` statements inside the try-block. Since this fixes the circular import problem, it is possible to run the modattrs.py script to check for changes. New changes since replacing wildcard imports, which are probably undesirable, are the removal of these attributes pointing to indirect Python submodules of the git module: base -> git.index.base fun -> git.index.fun head -> git.refs.head log -> git.refs.log reference -> git.refs.reference symbolic -> git.refs.symbolic tag -> git.refs.tag typ -> git.index.typ --- git/__init__.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index ebcaf89a2..7e07dcf13 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -119,27 +119,8 @@ from git.types import PathLike try: - from git.cmd import Git # @NoMove from git.compat import safe_decode # @NoMove from git.config import GitConfigParser # @NoMove - from git.db import GitCmdObjectDB, GitDB # @NoMove - from git.diff import ( # @NoMove - INDEX, - NULL_TREE, - Diff, - DiffConstants, - DiffIndex, - Diffable, - ) - from git.index import ( # @NoMove - BaseIndexEntry, - BlobFilter, - CheckoutError, - IndexEntry, - IndexFile, - StageType, - util, # noqa: F401 # For backward compatibility. - ) from git.objects import ( # @NoMove Blob, Commit, @@ -164,8 +145,27 @@ Tag, TagReference, ) - from git.remote import FetchInfo, PushInfo, Remote, RemoteProgress # @NoMove + from git.diff import ( # @NoMove + INDEX, + NULL_TREE, + Diff, + DiffConstants, + DiffIndex, + Diffable, + ) + from git.db import GitCmdObjectDB, GitDB # @NoMove + from git.cmd import Git # @NoMove from git.repo import Repo # @NoMove + from git.remote import FetchInfo, PushInfo, Remote, RemoteProgress # @NoMove + from git.index import ( # @NoMove + BaseIndexEntry, + BlobFilter, + CheckoutError, + IndexEntry, + IndexFile, + StageType, + util, # noqa: F401 # For backward compatibility. + ) from git.util import ( # @NoMove Actor, BlockingLockFile, From 31f89a1fa9f7fea7a13ba7aa9079364c22fb0e68 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 23:17:09 -0400 Subject: [PATCH 0881/1392] Add the nonpublic indirect submodule aliases back for now These should definitely never be used by code inside or outside of GitPython, as they have never been public, having even been omitted by the dynamic (and expansive) technique used to build git.__all__ in the past (which omitted modules intentionally). However, to ease compatibility, for now they are back. This is so that the change of making all imports explicit rather than using wildcards does not break anything. However, code using these names could never be relied on to work, and these should be considered eligible for removal, at least from the perspective of code outside GitPython. That is for the indirect submodules whose absence as a same-named direct submodule or attribute listed in __all__ was readily discoverable. The more difficult case is util. git.util is a module, git/util.py, which is how it is treated when it appears immediately after the "from" keyword, or as a key in sys.modules. However, the expression `git.util`, and a `from git import util` import, instead access the separate git.index.util module, which due to a wildcard import has been an attribute of the top-level git module for a long time. It may not be safe to change this, because although any code anywhere is better off not relying on this, this situation hasn't been (and isn't) immediately clear. To help with it somewhat, this also includes a detailed comment over where util is imported from git.index, explaining the situation. --- git/__init__.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/git/__init__.py b/git/__init__.py index 7e07dcf13..a13030456 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -144,6 +144,11 @@ SymbolicReference, Tag, TagReference, + head, # noqa: F401 # Nonpublic. May disappear! Use git.refs.head. + log, # noqa: F401 # Nonpublic. May disappear! Use git.refs.log. + reference, # noqa: F401 # Nonpublic. May disappear! Use git.refs.reference. + symbolic, # noqa: F401 # Nonpublic. May disappear! Use git.refs.symbolic. + tag, # noqa: F401 # Nonpublic. May disappear! Use git.refs.tag. ) from git.diff import ( # @NoMove INDEX, @@ -164,7 +169,21 @@ IndexEntry, IndexFile, StageType, - util, # noqa: F401 # For backward compatibility. + base, # noqa: F401 # Nonpublic. May disappear! Use git.index.base. + fun, # noqa: F401 # Nonpublic. May disappear! Use git.index.fun. + typ, # noqa: F401 # Nonpublic. May disappear! Use git.index.typ. + # + # NOTE: The expression `git.util` evaluates to git.index.util, and the import + # `from git import util` imports git.index.util, NOT git.util. It may not be + # feasible to change this until the next major version, to avoid breaking code + # inadvertently relying on it. If git.index.util really is what you want, use or + # import from that name, to avoid confusion. To use the "real" git.util module, + # write `from git.util import ...`, or access it as `sys.modules["git.util"]`. + # (This differs from other historical indirect-submodule imports that are + # unambiguously nonpublic and are subject to immediate removal. Here, the public + # git.util module, even though different, makes it less discoverable that the + # expression `git.util` refers to a non-public attribute of the git module.) + util, # noqa: F401 ) from git.util import ( # @NoMove Actor, From 9bbbcb5e847c9f0e4c91cb75d1c93be2b9cb1f57 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 23:28:09 -0400 Subject: [PATCH 0882/1392] Further improve git.objects.util module docstring --- git/objects/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/objects/util.py b/git/objects/util.py index 08aef132a..2cba89a9f 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -3,7 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""General utility functions for working with git objects.""" +"""Utility functions for working with git objects.""" __all__ = ( "get_object_type_by_name", From 00f4cbcf46426311f3cb6748912a1ea675639cf2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 23:32:06 -0400 Subject: [PATCH 0883/1392] Add missing submodule imports in git.objects Since they are listed in __all__. (They are imported either way because names are imported from them, and this caused them to be present with the same effect.) Though they are proably about to be removed along with the corresponding entries in __all__. --- git/objects/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/git/objects/__init__.py b/git/objects/__init__.py index 4f18c8754..d146901a1 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -23,6 +23,7 @@ "TreeModifier", ] +from . import base, blob, commit, submodule, tag, tree from .base import IndexObject, Object from .blob import Blob from .commit import Commit From fcc741838dbffa45648f3f224c01b9cb8941adc2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 23:35:58 -0400 Subject: [PATCH 0884/1392] Don't explicitly list direct submodules in __all__ This is for non-toplevel __all__, as git.__all__ already did not do this. As noted in some of the previous commit messags that added them, omitting them might be a bit safer in terms of the impact of bugs bugs in code using GitPython, in that unexpected modules, some of which have the same name as other modules within GitPython, won't be imported due to wildcard imports from intermediate subpackages (those that are below the top-level git package/module but collect non-subpackage modules). Though it is hard to know, since some of these would have been imported before, when an __all__ was not defined at that level. However, a separate benefit of omitting them is consistency with git.__all__, which does not list the direct Python submodules of the git module. This does not affect the output of the modattrs.py script, because the attributes exist with the same objects as their values as a result of those Python submodules being imported (in "from" imports and otherwise), including when importing the top-level git module. --- git/index/__init__.py | 5 ----- git/objects/__init__.py | 7 ------- git/objects/submodule/__init__.py | 11 +---------- git/refs/__init__.py | 7 ------- git/repo/__init__.py | 3 +-- 5 files changed, 2 insertions(+), 31 deletions(-) diff --git a/git/index/__init__.py b/git/index/__init__.py index 2086d67f9..ba48110fd 100644 --- a/git/index/__init__.py +++ b/git/index/__init__.py @@ -4,10 +4,6 @@ """Initialize the index package.""" __all__ = [ - "base", - "fun", - "typ", - "util", "BaseIndexEntry", "BlobFilter", "CheckoutError", @@ -16,6 +12,5 @@ "StageType", ] -from . import base, fun, typ, util from .base import CheckoutError, IndexFile from .typ import BaseIndexEntry, BlobFilter, IndexEntry, StageType diff --git a/git/objects/__init__.py b/git/objects/__init__.py index d146901a1..4447ca50d 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -4,12 +4,6 @@ """Import all submodules' main classes into the package space.""" __all__ = [ - "base", - "blob", - "commit", - "submodule", - "tag", - "tree", "IndexObject", "Object", "Blob", @@ -23,7 +17,6 @@ "TreeModifier", ] -from . import base, blob, commit, submodule, tag, tree from .base import IndexObject, Object from .blob import Blob from .commit import Commit diff --git a/git/objects/submodule/__init__.py b/git/objects/submodule/__init__.py index 383439545..c0604e76f 100644 --- a/git/objects/submodule/__init__.py +++ b/git/objects/submodule/__init__.py @@ -1,16 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -__all__ = [ - "base", - "root", - "util", - "Submodule", - "UpdateProgress", - "RootModule", - "RootUpdateProgress", -] +__all__ = ["Submodule", "UpdateProgress", "RootModule", "RootUpdateProgress"] -from . import base, root, util from .base import Submodule, UpdateProgress from .root import RootModule, RootUpdateProgress diff --git a/git/refs/__init__.py b/git/refs/__init__.py index 04279901a..d6157e6f3 100644 --- a/git/refs/__init__.py +++ b/git/refs/__init__.py @@ -2,12 +2,6 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ __all__ = [ - "head", - "log", - "reference", - "remote", - "symbolic", - "tag", "HEAD", "Head", "RefLog", @@ -19,7 +13,6 @@ "TagReference", ] -from . import head, log, reference, remote, symbolic, tag from .head import HEAD, Head from .log import RefLog, RefLogEntry from .reference import Reference diff --git a/git/repo/__init__.py b/git/repo/__init__.py index 3b784b975..66319ef95 100644 --- a/git/repo/__init__.py +++ b/git/repo/__init__.py @@ -3,7 +3,6 @@ """Initialize the repo package.""" -__all__ = ["base", "fun", "Repo"] +__all__ = ["Repo"] -from . import base, fun from .base import Repo From 78055a8b8473a25e8f36a3846b920a698e8ba66b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 18 Mar 2024 23:52:57 -0400 Subject: [PATCH 0885/1392] Pick a consistent type for __all__ (for now, list) This makes all __all__ everywhere in the git package lists. Before, roughly half were lists and half were tuples. There are reasonable theoretical arguments for both, and in practice all tools today support both. Traditionally using a list is far more common, and it remains at least somewhat more common. Furthermore, git/util.py uses a list and is currently written to append an element to it that is conditionally defined on Windows (though it would probably be fine for that to be the only list, since it reflects an actual relevant difference about it). The goal here is just to remove inconsistency that does not signify anything and is the result of drift over time. If a reason (even preference) arises to make them all tuples in the future, then that is also probably fine. --- git/cmd.py | 2 +- git/config.py | 2 +- git/db.py | 2 +- git/diff.py | 2 +- git/index/base.py | 2 +- git/index/fun.py | 4 ++-- git/index/typ.py | 2 +- git/index/util.py | 2 +- git/objects/base.py | 2 +- git/objects/blob.py | 2 +- git/objects/commit.py | 2 +- git/objects/fun.py | 4 ++-- git/objects/submodule/util.py | 4 ++-- git/objects/tag.py | 2 +- git/objects/tree.py | 2 +- git/objects/util.py | 4 ++-- git/remote.py | 2 +- git/repo/base.py | 2 +- git/repo/fun.py | 4 ++-- 19 files changed, 24 insertions(+), 24 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 7459fae97..03e5d7ffc 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -5,7 +5,7 @@ from __future__ import annotations -__all__ = ("Git",) +__all__ = ["Git"] import contextlib import io diff --git a/git/config.py b/git/config.py index 1c9761c1b..3ce9b123f 100644 --- a/git/config.py +++ b/git/config.py @@ -5,7 +5,7 @@ """Parser for reading and writing configuration files.""" -__all__ = ("GitConfigParser", "SectionConstraint") +__all__ = ["GitConfigParser", "SectionConstraint"] import abc import configparser as cp diff --git a/git/db.py b/git/db.py index 302192bdf..cacd030d0 100644 --- a/git/db.py +++ b/git/db.py @@ -3,7 +3,7 @@ """Module with our own gitdb implementation - it uses the git command.""" -__all__ = ("GitCmdObjectDB", "GitDB") +__all__ = ["GitCmdObjectDB", "GitDB"] from gitdb.base import OInfo, OStream from gitdb.db import GitDB, LooseObjectDB diff --git a/git/diff.py b/git/diff.py index 1381deca0..0e39fe7a8 100644 --- a/git/diff.py +++ b/git/diff.py @@ -3,7 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -__all__ = ("DiffConstants", "NULL_TREE", "INDEX", "Diffable", "DiffIndex", "Diff") +__all__ = ["DiffConstants", "NULL_TREE", "INDEX", "Diffable", "DiffIndex", "Diff"] import enum import re diff --git a/git/index/base.py b/git/index/base.py index b49841435..b8161ea52 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -6,7 +6,7 @@ """Module containing :class:`IndexFile`, an Index implementation facilitating all kinds of index manipulations such as querying and merging.""" -__all__ = ("IndexFile", "CheckoutError", "StageType") +__all__ = ["IndexFile", "CheckoutError", "StageType"] import contextlib import datetime diff --git a/git/index/fun.py b/git/index/fun.py index b24e803a3..59cce6ae6 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -4,7 +4,7 @@ """Standalone functions to accompany the index implementation and make it more versatile.""" -__all__ = ( +__all__ = [ "write_cache", "read_cache", "write_tree_from_cache", @@ -13,7 +13,7 @@ "S_IFGITLINK", "run_commit_hook", "hook_path", -) +] from io import BytesIO import os diff --git a/git/index/typ.py b/git/index/typ.py index 0fbcd69f0..974252528 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -3,7 +3,7 @@ """Additional types used by the index.""" -__all__ = ("BlobFilter", "BaseIndexEntry", "IndexEntry", "StageType") +__all__ = ["BlobFilter", "BaseIndexEntry", "IndexEntry", "StageType"] from binascii import b2a_hex from pathlib import Path diff --git a/git/index/util.py b/git/index/util.py index 4aee61bce..e59cb609f 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -3,7 +3,7 @@ """Index utilities.""" -__all__ = ("TemporaryFileSwap", "post_clear_cache", "default_index", "git_working_dir") +__all__ = ["TemporaryFileSwap", "post_clear_cache", "default_index", "git_working_dir"] import contextlib from functools import wraps diff --git a/git/objects/base.py b/git/objects/base.py index 07ff639e5..eeaebc09b 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -3,7 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -__all__ = ("Object", "IndexObject") +__all__ = ["Object", "IndexObject"] import os.path as osp diff --git a/git/objects/blob.py b/git/objects/blob.py index 50500f550..58de59642 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -3,7 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -__all__ = ("Blob",) +__all__ = ["Blob"] from mimetypes import guess_type import sys diff --git a/git/objects/commit.py b/git/objects/commit.py index b22308726..8de52980c 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -3,7 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -__all__ = ("Commit",) +__all__ = ["Commit"] from collections import defaultdict import datetime diff --git a/git/objects/fun.py b/git/objects/fun.py index 9bd2d8f10..fe57da13a 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -3,12 +3,12 @@ """Functions that are supposed to be as fast as possible.""" -__all__ = ( +__all__ = [ "tree_to_stream", "tree_entries_from_data", "traverse_trees_recursive", "traverse_tree_recursive", -) +] from stat import S_ISDIR diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index 3a09d6a17..c021510d8 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -1,13 +1,13 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -__all__ = ( +__all__ = [ "sm_section", "sm_name", "mkhead", "find_first_remote_branch", "SubmoduleConfigParser", -) +] from io import BytesIO import weakref diff --git a/git/objects/tag.py b/git/objects/tag.py index 5ad311590..a3bb0b882 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -9,7 +9,7 @@ For lightweight tags, see the :mod:`git.refs.tag` module. """ -__all__ = ("TagObject",) +__all__ = ["TagObject"] import sys diff --git a/git/objects/tree.py b/git/objects/tree.py index d8320b319..09184a781 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -3,7 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -__all__ = ("TreeModifier", "Tree") +__all__ = ["TreeModifier", "Tree"] import sys diff --git a/git/objects/util.py b/git/objects/util.py index 2cba89a9f..5c56e6134 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -5,7 +5,7 @@ """Utility functions for working with git objects.""" -__all__ = ( +__all__ = [ "get_object_type_by_name", "parse_date", "parse_actor_and_date", @@ -17,7 +17,7 @@ "Actor", "tzoffset", "utc", -) +] from abc import ABC, abstractmethod import calendar diff --git a/git/remote.py b/git/remote.py index 54a5c459c..f2ecd0f36 100644 --- a/git/remote.py +++ b/git/remote.py @@ -5,7 +5,7 @@ """Module implementing a remote object allowing easy access to git remotes.""" -__all__ = ("RemoteProgress", "PushInfo", "FetchInfo", "Remote") +__all__ = ["RemoteProgress", "PushInfo", "FetchInfo", "Remote"] import contextlib import logging diff --git a/git/repo/base.py b/git/repo/base.py index 63b21fdbf..51ea76901 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -5,7 +5,7 @@ from __future__ import annotations -__all__ = ("Repo",) +__all__ = ["Repo"] import gc import logging diff --git a/git/repo/fun.py b/git/repo/fun.py index 1defcb1ac..e44d9c644 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -5,7 +5,7 @@ from __future__ import annotations -__all__ = ( +__all__ = [ "rev_parse", "is_git_dir", "touch", @@ -15,7 +15,7 @@ "deref_tag", "to_commit", "find_worktree_git_dir", -) +] import os import os.path as osp From ecdb6aa25a95e2cb3a650a26f3ffd9c45e127ed1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 19 Mar 2024 00:05:15 -0400 Subject: [PATCH 0886/1392] Save diff of non-__all__ attributes across import changes This commits the diff of the output of the modattrs.py script between when the script was introduced in 1e5a944, and now (with the import refactoring finished and no wildcard imports remaining outside the test suite, and only there in one import statement for test helpers). Neither this diff nor modattrs.py itself will be retained. Both will be removed in the next commit. This is committed here to facilitate inspection and debugging (especially if turns out there are thus-far undetected regressions). --- ab.diff | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 ab.diff diff --git a/ab.diff b/ab.diff new file mode 100644 index 000000000..1f2885b90 --- /dev/null +++ b/ab.diff @@ -0,0 +1,42 @@ +diff --git a/a b/b +index 81b3f984..099fa33b 100644 +--- a/a ++++ b/b +@@ -634,7 +634,6 @@ git.objects: + blob: + commit: + fun: +- inspect: + submodule: + tag: + tree: +@@ -770,6 +769,10 @@ git.objects.fun: + + + git.objects.submodule: ++ RootModule: ++ RootUpdateProgress: ++ Submodule: ++ UpdateProgress: + __builtins__: {'__name__': 'builtins', '__doc__': "Built-in functions, types, exceptions, and other objects.\n\nThis module provides direct access to all 'built-in'\nidentifiers of Python; for example, builtins.len is\nthe full name for the built-in function len().\n\nThis module is not normally accessed explicitly by most\napplications, but can be useful in modules that provide\nobjects with the same name as a built-in value, but in\nwhich the built-in of that name is also needed.", '__package__': '', '__loader__': , '__spec__': ModuleSpec(name='builtins', loader=, origin='built-in'), '__build_class__': , '__import__': , 'abs': , 'all': , 'any': , 'ascii': , 'bin': , 'breakpoint': , 'callable': , 'chr': , 'compile': , 'delattr': , 'dir': , 'divmod': , 'eval': , 'exec': , 'format': , 'getattr': , 'globals': , 'hasattr': , 'hash': , 'hex': , 'id': , 'input': , 'isinstance': , 'issubclass': , 'iter': , 'aiter': , 'len': , 'locals': , 'max': , 'min': , 'next': , 'anext': , 'oct': , 'ord': , 'pow': , 'print': , 'repr': , 'round': , 'setattr': , 'sorted': , 'sum': , 'vars': , 'None': None, 'Ellipsis': Ellipsis, 'NotImplemented': NotImplemented, 'False': False, 'True': True, 'bool': , 'memoryview': , 'bytearray': , 'bytes': , 'classmethod': , 'complex': , 'dict': , 'enumerate': , 'filter': , 'float': , 'frozenset': , 'property': , 'int': , 'list': , 'map': , 'object': , 'range': , 'reversed': , 'set': , 'slice': , 'staticmethod': , 'str': , 'super': , 'tuple': , 'type': , 'zip': , '__debug__': True, 'BaseException': , 'BaseExceptionGroup': , 'Exception': , 'GeneratorExit': , 'KeyboardInterrupt': , 'SystemExit': , 'ArithmeticError': , 'AssertionError': , 'AttributeError': , 'BufferError': , 'EOFError': , 'ImportError': , 'LookupError': , 'MemoryError': , 'NameError': , 'OSError': , 'ReferenceError': , 'RuntimeError': , 'StopAsyncIteration': , 'StopIteration': , 'SyntaxError': , 'SystemError': , 'TypeError': , 'ValueError': , 'Warning': , 'FloatingPointError': , 'OverflowError': , 'ZeroDivisionError': , 'BytesWarning': , 'DeprecationWarning': , 'EncodingWarning': , 'FutureWarning': , 'ImportWarning': , 'PendingDeprecationWarning': , 'ResourceWarning': , 'RuntimeWarning': , 'SyntaxWarning': , 'UnicodeWarning': , 'UserWarning': , 'BlockingIOError': , 'ChildProcessError': , 'ConnectionError': , 'FileExistsError': , 'FileNotFoundError': , 'InterruptedError': , 'IsADirectoryError': , 'NotADirectoryError': , 'PermissionError': , 'ProcessLookupError': , 'TimeoutError': , 'IndentationError': , 'IndexError': , 'KeyError': , 'ModuleNotFoundError': , 'NotImplementedError': , 'RecursionError': , 'UnboundLocalError': , 'UnicodeError': , 'BrokenPipeError': , 'ConnectionAbortedError': , 'ConnectionRefusedError': , 'ConnectionResetError': , 'TabError': , 'UnicodeDecodeError': , 'UnicodeEncodeError': , 'UnicodeTranslateError': , 'ExceptionGroup': , 'EnvironmentError': , 'IOError': , 'WindowsError': , 'open': , 'quit': Use quit() or Ctrl-Z plus Return to exit, 'exit': Use exit() or Ctrl-Z plus Return to exit, 'copyright': Copyright (c) 2001-2023 Python Software Foundation.?All Rights Reserved.??Copyright (c) 2000 BeOpen.com.?All Rights Reserved.??Copyright (c) 1995-2001 Corporation for National Research Initiatives.?All Rights Reserved.??Copyright (c) 1991-1995 Stichting Mathematisch Centrum, Amsterdam.?All Rights Reserved., 'credits': Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands? for supporting Python development. See www.python.org for more information., 'license': Type license() to see the full license text, 'help': Type help() for interactive help, or help(object) for help about object.} + __cached__: 'C:\\Users\\ek\\source\\repos\\GitPython\\git\\objects\\submodule\\__pycache__\\__init__.cpython-312.pyc' + __doc__: None +@@ -881,9 +884,7 @@ git.objects.submodule.util: + Any: typing.Any + BytesIO: + GitConfigParser: +- IndexObject: + InvalidGitRepositoryError: +- Object: + PathLike: typing.Union[str, ForwardRef('os.PathLike[str]')] + Sequence: typing.Sequence + SubmoduleConfigParser: +@@ -998,7 +999,7 @@ git.objects.util: + ZERO: datetime.timedelta(0) + __builtins__: {'__name__': 'builtins', '__doc__': "Built-in functions, types, exceptions, and other objects.\n\nThis module provides direct access to all 'built-in'\nidentifiers of Python; for example, builtins.len is\nthe full name for the built-in function len().\n\nThis module is not normally accessed explicitly by most\napplications, but can be useful in modules that provide\nobjects with the same name as a built-in value, but in\nwhich the built-in of that name is also needed.", '__package__': '', '__loader__': , '__spec__': ModuleSpec(name='builtins', loader=, origin='built-in'), '__build_class__': , '__import__': , 'abs': , 'all': , 'any': , 'ascii': , 'bin': , 'breakpoint': , 'callable': , 'chr': , 'compile': , 'delattr': , 'dir': , 'divmod': , 'eval': , 'exec': , 'format': , 'getattr': , 'globals': , 'hasattr': , 'hash': , 'hex': , 'id': , 'input': , 'isinstance': , 'issubclass': , 'iter': , 'aiter': , 'len': , 'locals': , 'max': , 'min': , 'next': , 'anext': , 'oct': , 'ord': , 'pow': , 'print': , 'repr': , 'round': , 'setattr': , 'sorted': , 'sum': , 'vars': , 'None': None, 'Ellipsis': Ellipsis, 'NotImplemented': NotImplemented, 'False': False, 'True': True, 'bool': , 'memoryview': , 'bytearray': , 'bytes': , 'classmethod': , 'complex': , 'dict': , 'enumerate': , 'filter': , 'float': , 'frozenset': , 'property': , 'int': , 'list': , 'map': , 'object': , 'range': , 'reversed': , 'set': , 'slice': , 'staticmethod': , 'str': , 'super': , 'tuple': , 'type': , 'zip': , '__debug__': True, 'BaseException': , 'BaseExceptionGroup': , 'Exception': , 'GeneratorExit': , 'KeyboardInterrupt': , 'SystemExit': , 'ArithmeticError': , 'AssertionError': , 'AttributeError': , 'BufferError': , 'EOFError': , 'ImportError': , 'LookupError': , 'MemoryError': , 'NameError': , 'OSError': , 'ReferenceError': , 'RuntimeError': , 'StopAsyncIteration': , 'StopIteration': , 'SyntaxError': , 'SystemError': , 'TypeError': , 'ValueError': , 'Warning': , 'FloatingPointError': , 'OverflowError': , 'ZeroDivisionError': , 'BytesWarning': , 'DeprecationWarning': , 'EncodingWarning': , 'FutureWarning': , 'ImportWarning': , 'PendingDeprecationWarning': , 'ResourceWarning': , 'RuntimeWarning': , 'SyntaxWarning': , 'UnicodeWarning': , 'UserWarning': , 'BlockingIOError': , 'ChildProcessError': , 'ConnectionError': , 'FileExistsError': , 'FileNotFoundError': , 'InterruptedError': , 'IsADirectoryError': , 'NotADirectoryError': , 'PermissionError': , 'ProcessLookupError': , 'TimeoutError': , 'IndentationError': , 'IndexError': , 'KeyError': , 'ModuleNotFoundError': , 'NotImplementedError': , 'RecursionError': , 'UnboundLocalError': , 'UnicodeError': , 'BrokenPipeError': , 'ConnectionAbortedError': , 'ConnectionRefusedError': , 'ConnectionResetError': , 'TabError': , 'UnicodeDecodeError': , 'UnicodeEncodeError': , 'UnicodeTranslateError': , 'ExceptionGroup': , 'EnvironmentError': , 'IOError': , 'WindowsError': , 'open': , 'quit': Use quit() or Ctrl-Z plus Return to exit, 'exit': Use exit() or Ctrl-Z plus Return to exit, 'copyright': Copyright (c) 2001-2023 Python Software Foundation.?All Rights Reserved.??Copyright (c) 2000 BeOpen.com.?All Rights Reserved.??Copyright (c) 1995-2001 Corporation for National Research Initiatives.?All Rights Reserved.??Copyright (c) 1991-1995 Stichting Mathematisch Centrum, Amsterdam.?All Rights Reserved., 'credits': Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands? for supporting Python development. See www.python.org for more information., 'license': Type license() to see the full license text, 'help': Type help() for interactive help, or help(object) for help about object.} + __cached__: 'C:\\Users\\ek\\source\\repos\\GitPython\\git\\objects\\__pycache__\\util.cpython-312.pyc' +- __doc__: 'General utility functions.' ++ __doc__: 'Utility functions for working with git objects.' + __file__: 'C:\\Users\\ek\\source\\repos\\GitPython\\git\\objects\\util.py' + __loader__: <_frozen_importlib_external.SourceFileLoader object at 0x...> + __name__: 'git.objects.util' From f705fd6dde6047c7787809aeb691988fa4af80d8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 19 Mar 2024 00:15:50 -0400 Subject: [PATCH 0887/1392] Remove modattrs.py and related Now that it has served its purpose. (Of course, it can also be brought back from the history at any time if needed.) Changes: - Delete the modattrs.py script that was used to check for unintended changes to what module attributes existed and what objects they referred to, while doing the import refactoring. - Delete the ab.diff file showing the overall diff, which was temporarily introduced in the previous commit. - Remove the "a" and "b" temporary entries in .gitignore that were used to facilitate efficient use of modattrs.py while carrying out the import refactoring. --- .gitignore | 4 ---- ab.diff | 42 ------------------------------------------ modattrs.py | 53 ----------------------------------------------------- 3 files changed, 99 deletions(-) delete mode 100644 ab.diff delete mode 100755 modattrs.py diff --git a/.gitignore b/.gitignore index 1be4f3201..7765293d8 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,3 @@ output.txt # Finder metadata .DS_Store - -# Output files for modattrs.py (these entries will be removed soon) -a -b diff --git a/ab.diff b/ab.diff deleted file mode 100644 index 1f2885b90..000000000 --- a/ab.diff +++ /dev/null @@ -1,42 +0,0 @@ -diff --git a/a b/b -index 81b3f984..099fa33b 100644 ---- a/a -+++ b/b -@@ -634,7 +634,6 @@ git.objects: - blob: - commit: - fun: -- inspect: - submodule: - tag: - tree: -@@ -770,6 +769,10 @@ git.objects.fun: - - - git.objects.submodule: -+ RootModule: -+ RootUpdateProgress: -+ Submodule: -+ UpdateProgress: - __builtins__: {'__name__': 'builtins', '__doc__': "Built-in functions, types, exceptions, and other objects.\n\nThis module provides direct access to all 'built-in'\nidentifiers of Python; for example, builtins.len is\nthe full name for the built-in function len().\n\nThis module is not normally accessed explicitly by most\napplications, but can be useful in modules that provide\nobjects with the same name as a built-in value, but in\nwhich the built-in of that name is also needed.", '__package__': '', '__loader__': , '__spec__': ModuleSpec(name='builtins', loader=, origin='built-in'), '__build_class__': , '__import__': , 'abs': , 'all': , 'any': , 'ascii': , 'bin': , 'breakpoint': , 'callable': , 'chr': , 'compile': , 'delattr': , 'dir': , 'divmod': , 'eval': , 'exec': , 'format': , 'getattr': , 'globals': , 'hasattr': , 'hash': , 'hex': , 'id': , 'input': , 'isinstance': , 'issubclass': , 'iter': , 'aiter': , 'len': , 'locals': , 'max': , 'min': , 'next': , 'anext': , 'oct': , 'ord': , 'pow': , 'print': , 'repr': , 'round': , 'setattr': , 'sorted': , 'sum': , 'vars': , 'None': None, 'Ellipsis': Ellipsis, 'NotImplemented': NotImplemented, 'False': False, 'True': True, 'bool': , 'memoryview': , 'bytearray': , 'bytes': , 'classmethod': , 'complex': , 'dict': , 'enumerate': , 'filter': , 'float': , 'frozenset': , 'property': , 'int': , 'list': , 'map': , 'object': , 'range': , 'reversed': , 'set': , 'slice': , 'staticmethod': , 'str': , 'super': , 'tuple': , 'type': , 'zip': , '__debug__': True, 'BaseException': , 'BaseExceptionGroup': , 'Exception': , 'GeneratorExit': , 'KeyboardInterrupt': , 'SystemExit': , 'ArithmeticError': , 'AssertionError': , 'AttributeError': , 'BufferError': , 'EOFError': , 'ImportError': , 'LookupError': , 'MemoryError': , 'NameError': , 'OSError': , 'ReferenceError': , 'RuntimeError': , 'StopAsyncIteration': , 'StopIteration': , 'SyntaxError': , 'SystemError': , 'TypeError': , 'ValueError': , 'Warning': , 'FloatingPointError': , 'OverflowError': , 'ZeroDivisionError': , 'BytesWarning': , 'DeprecationWarning': , 'EncodingWarning': , 'FutureWarning': , 'ImportWarning': , 'PendingDeprecationWarning': , 'ResourceWarning': , 'RuntimeWarning': , 'SyntaxWarning': , 'UnicodeWarning': , 'UserWarning': , 'BlockingIOError': , 'ChildProcessError': , 'ConnectionError': , 'FileExistsError': , 'FileNotFoundError': , 'InterruptedError': , 'IsADirectoryError': , 'NotADirectoryError': , 'PermissionError': , 'ProcessLookupError': , 'TimeoutError': , 'IndentationError': , 'IndexError': , 'KeyError': , 'ModuleNotFoundError': , 'NotImplementedError': , 'RecursionError': , 'UnboundLocalError': , 'UnicodeError': , 'BrokenPipeError': , 'ConnectionAbortedError': , 'ConnectionRefusedError': , 'ConnectionResetError': , 'TabError': , 'UnicodeDecodeError': , 'UnicodeEncodeError': , 'UnicodeTranslateError': , 'ExceptionGroup': , 'EnvironmentError': , 'IOError': , 'WindowsError': , 'open': , 'quit': Use quit() or Ctrl-Z plus Return to exit, 'exit': Use exit() or Ctrl-Z plus Return to exit, 'copyright': Copyright (c) 2001-2023 Python Software Foundation.?All Rights Reserved.??Copyright (c) 2000 BeOpen.com.?All Rights Reserved.??Copyright (c) 1995-2001 Corporation for National Research Initiatives.?All Rights Reserved.??Copyright (c) 1991-1995 Stichting Mathematisch Centrum, Amsterdam.?All Rights Reserved., 'credits': Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands? for supporting Python development. See www.python.org for more information., 'license': Type license() to see the full license text, 'help': Type help() for interactive help, or help(object) for help about object.} - __cached__: 'C:\\Users\\ek\\source\\repos\\GitPython\\git\\objects\\submodule\\__pycache__\\__init__.cpython-312.pyc' - __doc__: None -@@ -881,9 +884,7 @@ git.objects.submodule.util: - Any: typing.Any - BytesIO: - GitConfigParser: -- IndexObject: - InvalidGitRepositoryError: -- Object: - PathLike: typing.Union[str, ForwardRef('os.PathLike[str]')] - Sequence: typing.Sequence - SubmoduleConfigParser: -@@ -998,7 +999,7 @@ git.objects.util: - ZERO: datetime.timedelta(0) - __builtins__: {'__name__': 'builtins', '__doc__': "Built-in functions, types, exceptions, and other objects.\n\nThis module provides direct access to all 'built-in'\nidentifiers of Python; for example, builtins.len is\nthe full name for the built-in function len().\n\nThis module is not normally accessed explicitly by most\napplications, but can be useful in modules that provide\nobjects with the same name as a built-in value, but in\nwhich the built-in of that name is also needed.", '__package__': '', '__loader__': , '__spec__': ModuleSpec(name='builtins', loader=, origin='built-in'), '__build_class__': , '__import__': , 'abs': , 'all': , 'any': , 'ascii': , 'bin': , 'breakpoint': , 'callable': , 'chr': , 'compile': , 'delattr': , 'dir': , 'divmod': , 'eval': , 'exec': , 'format': , 'getattr': , 'globals': , 'hasattr': , 'hash': , 'hex': , 'id': , 'input': , 'isinstance': , 'issubclass': , 'iter': , 'aiter': , 'len': , 'locals': , 'max': , 'min': , 'next': , 'anext': , 'oct': , 'ord': , 'pow': , 'print': , 'repr': , 'round': , 'setattr': , 'sorted': , 'sum': , 'vars': , 'None': None, 'Ellipsis': Ellipsis, 'NotImplemented': NotImplemented, 'False': False, 'True': True, 'bool': , 'memoryview': , 'bytearray': , 'bytes': , 'classmethod': , 'complex': , 'dict': , 'enumerate': , 'filter': , 'float': , 'frozenset': , 'property': , 'int': , 'list': , 'map': , 'object': , 'range': , 'reversed': , 'set': , 'slice': , 'staticmethod': , 'str': , 'super': , 'tuple': , 'type': , 'zip': , '__debug__': True, 'BaseException': , 'BaseExceptionGroup': , 'Exception': , 'GeneratorExit': , 'KeyboardInterrupt': , 'SystemExit': , 'ArithmeticError': , 'AssertionError': , 'AttributeError': , 'BufferError': , 'EOFError': , 'ImportError': , 'LookupError': , 'MemoryError': , 'NameError': , 'OSError': , 'ReferenceError': , 'RuntimeError': , 'StopAsyncIteration': , 'StopIteration': , 'SyntaxError': , 'SystemError': , 'TypeError': , 'ValueError': , 'Warning': , 'FloatingPointError': , 'OverflowError': , 'ZeroDivisionError': , 'BytesWarning': , 'DeprecationWarning': , 'EncodingWarning': , 'FutureWarning': , 'ImportWarning': , 'PendingDeprecationWarning': , 'ResourceWarning': , 'RuntimeWarning': , 'SyntaxWarning': , 'UnicodeWarning': , 'UserWarning': , 'BlockingIOError': , 'ChildProcessError': , 'ConnectionError': , 'FileExistsError': , 'FileNotFoundError': , 'InterruptedError': , 'IsADirectoryError': , 'NotADirectoryError': , 'PermissionError': , 'ProcessLookupError': , 'TimeoutError': , 'IndentationError': , 'IndexError': , 'KeyError': , 'ModuleNotFoundError': , 'NotImplementedError': , 'RecursionError': , 'UnboundLocalError': , 'UnicodeError': , 'BrokenPipeError': , 'ConnectionAbortedError': , 'ConnectionRefusedError': , 'ConnectionResetError': , 'TabError': , 'UnicodeDecodeError': , 'UnicodeEncodeError': , 'UnicodeTranslateError': , 'ExceptionGroup': , 'EnvironmentError': , 'IOError': , 'WindowsError': , 'open': , 'quit': Use quit() or Ctrl-Z plus Return to exit, 'exit': Use exit() or Ctrl-Z plus Return to exit, 'copyright': Copyright (c) 2001-2023 Python Software Foundation.?All Rights Reserved.??Copyright (c) 2000 BeOpen.com.?All Rights Reserved.??Copyright (c) 1995-2001 Corporation for National Research Initiatives.?All Rights Reserved.??Copyright (c) 1991-1995 Stichting Mathematisch Centrum, Amsterdam.?All Rights Reserved., 'credits': Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands? for supporting Python development. See www.python.org for more information., 'license': Type license() to see the full license text, 'help': Type help() for interactive help, or help(object) for help about object.} - __cached__: 'C:\\Users\\ek\\source\\repos\\GitPython\\git\\objects\\__pycache__\\util.cpython-312.pyc' -- __doc__: 'General utility functions.' -+ __doc__: 'Utility functions for working with git objects.' - __file__: 'C:\\Users\\ek\\source\\repos\\GitPython\\git\\objects\\util.py' - __loader__: <_frozen_importlib_external.SourceFileLoader object at 0x...> - __name__: 'git.objects.util' diff --git a/modattrs.py b/modattrs.py deleted file mode 100755 index 245f68912..000000000 --- a/modattrs.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python - -"""Script to get the names and "stabilized" reprs of module attributes in GitPython. - -Run with :envvar:`PYTHONHASHSEED` set to ``0`` for fully comparable results. These are -only still meaningful for comparing if the same platform and Python version are used. - -The output of this script should probably not be committed, because within the reprs of -objects found in modules, it may contain sensitive information, such as API keys stored -in environment variables. The "sanitization" performed here is only for common forms of -whitespace that clash with the output format. -""" - -# fmt: off - -__all__ = ["git", "main"] - -import itertools -import re -import sys - -import git - - -def main(): - # This assumes `import git` causes all of them to be loaded. - gitpython_modules = sorted( - (module_name, module) - for module_name, module in sys.modules.items() - if re.match(r"git(?:\.|$)", module_name) - ) - - # We will print two blank lines between successive module reports. - separators = itertools.chain(("",), itertools.repeat("\n\n")) - - # Report each module's contents. - for (module_name, module), separator in zip(gitpython_modules, separators): - print(f"{separator}{module_name}:") - - attributes = sorted( - (name, value) - for name, value in module.__dict__.items() - if name != "__all__" # Because we are deliberately adding these. - ) - - for name, value in attributes: - sanitized_repr = re.sub(r"[\r\n\v\f]", "?", repr(value)) - normalized_repr = re.sub(r" at 0x[0-9a-fA-F]+", " at 0x...", sanitized_repr) - print(f" {name}: {normalized_repr}") - - -if __name__ == "__main__": - main() From 4a4d880fec4364c7d49e7708238a962942ae69f3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 19 Mar 2024 00:45:40 -0400 Subject: [PATCH 0888/1392] Improve test suite import grouping/sorting, __all__ placement There is only one __all__ in the test suite, so this is mostly the change to imports, grouping and sorting them in a fully consistent style that is the same as the import style in the code under test. --- test/lib/helper.py | 32 ++++++++++++++++---------------- test/performance/__init__.py | 2 ++ test/performance/lib.py | 5 +++-- test/performance/test_commit.py | 6 ++++-- test/performance/test_odb.py | 2 +- test/performance/test_streams.py | 8 ++++---- test/test_actor.py | 3 ++- test/test_base.py | 10 +++++----- test/test_blob.py | 3 ++- test/test_clone.py | 5 +---- test/test_commit.py | 20 +++++++++++--------- test/test_config.py | 2 +- test/test_db.py | 5 +++-- test/test_diff.py | 1 + test/test_docs.py | 3 +-- test/test_exc.py | 7 ++++--- test/test_fun.py | 17 ++++++++--------- test/test_git.py | 3 ++- test/test_index.py | 14 ++++---------- test/test_reflog.py | 5 +++-- test/test_refs.py | 23 ++++++++++++----------- test/test_remote.py | 3 +-- test/test_repo.py | 7 ++----- test/test_stats.py | 3 ++- test/test_submodule.py | 1 + test/test_tree.py | 3 ++- test/test_util.py | 1 + 27 files changed, 99 insertions(+), 95 deletions(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index 45a778b7d..5d91447ea 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -3,6 +3,22 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +__all__ = [ + "fixture_path", + "fixture", + "StringProcessAdapter", + "with_rw_directory", + "with_rw_repo", + "with_rw_and_rw_remote_repo", + "TestBase", + "VirtualEnvironment", + "TestCase", + "SkipTest", + "skipIf", + "GIT_REPO", + "GIT_DAEMON_PORT", +] + import contextlib from functools import wraps import gc @@ -31,22 +47,6 @@ GIT_REPO = os.environ.get("GIT_PYTHON_TEST_GIT_REPO_BASE", ospd(ospd(ospd(__file__)))) GIT_DAEMON_PORT = os.environ.get("GIT_PYTHON_TEST_GIT_DAEMON_PORT", "19418") -__all__ = ( - "fixture_path", - "fixture", - "StringProcessAdapter", - "with_rw_directory", - "with_rw_repo", - "with_rw_and_rw_remote_repo", - "TestBase", - "VirtualEnvironment", - "TestCase", - "SkipTest", - "skipIf", - "GIT_REPO", - "GIT_DAEMON_PORT", -) - _logger = logging.getLogger(__name__) # { Routines diff --git a/test/performance/__init__.py b/test/performance/__init__.py index e69de29bb..56b5d89db 100644 --- a/test/performance/__init__.py +++ b/test/performance/__init__.py @@ -0,0 +1,2 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/test/performance/lib.py b/test/performance/lib.py index d08e1027f..2ca3c409b 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -5,13 +5,14 @@ import logging import os +import os.path as osp import tempfile from git import Repo from git.db import GitCmdObjectDB, GitDB -from test.lib import TestBase from git.util import rmtree -import os.path as osp + +from test.lib import TestBase # { Invariants diff --git a/test/performance/test_commit.py b/test/performance/test_commit.py index 00d768f0a..b943f1975 100644 --- a/test/performance/test_commit.py +++ b/test/performance/test_commit.py @@ -10,9 +10,11 @@ from time import time import sys -from .lib import TestBigRepoRW -from git import Commit from gitdb import IStream + +from git import Commit + +from test.performance.lib import TestBigRepoRW from test.test_commit import TestCommitSerialization diff --git a/test/performance/test_odb.py b/test/performance/test_odb.py index 00e245fb7..fdbbeb8c3 100644 --- a/test/performance/test_odb.py +++ b/test/performance/test_odb.py @@ -6,7 +6,7 @@ import sys from time import time -from .lib import TestBigRepoR +from test.performance.lib import TestBigRepoR class TestObjDBPerformance(TestBigRepoR): diff --git a/test/performance/test_streams.py b/test/performance/test_streams.py index 56b5274ec..f6ffeba8e 100644 --- a/test/performance/test_streams.py +++ b/test/performance/test_streams.py @@ -5,18 +5,18 @@ import gc import os +import os.path as osp import subprocess import sys from time import time -from test.lib import with_rw_repo -from git.util import bin_to_hex from gitdb import LooseObjectDB, IStream from gitdb.test.lib import make_memory_file -import os.path as osp +from git.util import bin_to_hex -from .lib import TestBigRepoR +from test.lib import with_rw_repo +from test.performance.lib import TestBigRepoR class TestObjDBPerformance(TestBigRepoR): diff --git a/test/test_actor.py b/test/test_actor.py index caf095739..5e6635709 100644 --- a/test/test_actor.py +++ b/test/test_actor.py @@ -3,9 +3,10 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from test.lib import TestBase from git import Actor +from test.lib import TestBase + class TestActor(TestBase): def test_from_string_should_separate_name_and_email(self): diff --git a/test/test_base.py b/test/test_base.py index e477b4837..86bcc5c79 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -5,18 +5,18 @@ import gc import os +import os.path as osp import sys import tempfile from unittest import skipIf from git import Repo -from git.objects import Blob, Tree, Commit, TagObject +from git.objects import Blob, Commit, TagObject, Tree +import git.objects.base as base from git.objects.util import get_object_type_by_name -from test.lib import TestBase as _TestBase, with_rw_repo, with_rw_and_rw_remote_repo -from git.util import hex_to_bin, HIDE_WINDOWS_FREEZE_ERRORS +from git.util import HIDE_WINDOWS_FREEZE_ERRORS, hex_to_bin -import git.objects.base as base -import os.path as osp +from test.lib import TestBase as _TestBase, with_rw_and_rw_remote_repo, with_rw_repo class TestBase(_TestBase): diff --git a/test/test_blob.py b/test/test_blob.py index ff59c67ea..affaa60fc 100644 --- a/test/test_blob.py +++ b/test/test_blob.py @@ -3,9 +3,10 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from test.lib import TestBase from git import Blob +from test.lib import TestBase + class TestBlob(TestBase): def test_mime_type_should_return_mime_type_for_known_types(self): diff --git a/test/test_clone.py b/test/test_clone.py index be2e6b19b..126ef0063 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -6,10 +6,7 @@ import git -from .lib import ( - TestBase, - with_rw_directory, -) +from test.lib import TestBase, with_rw_directory class TestClone(TestBase): diff --git a/test/test_commit.py b/test/test_commit.py index 5571b9ecb..5832258de 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -6,23 +6,25 @@ import copy from datetime import datetime from io import BytesIO +import os.path as osp import re import sys import time from unittest.mock import Mock -from git import ( - Commit, - Actor, -) -from git import Repo +from gitdb import IStream + +from git import Actor, Commit, Repo from git.objects.util import tzoffset, utc from git.repo.fun import touch -from test.lib import TestBase, with_rw_repo, fixture_path, StringProcessAdapter -from test.lib import with_rw_directory -from gitdb import IStream -import os.path as osp +from test.lib import ( + StringProcessAdapter, + TestBase, + fixture_path, + with_rw_directory, + with_rw_repo, +) class TestCommitSerialization(TestBase): diff --git a/test/test_config.py b/test/test_config.py index ac19a7fa8..0911d0262 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -15,8 +15,8 @@ from git import GitConfigParser from git.config import _OMD, cp from git.util import rmfile -from test.lib import SkipTest, TestCase, fixture_path, with_rw_directory +from test.lib import SkipTest, TestCase, fixture_path, with_rw_directory _tc_lock_fpaths = osp.join(osp.dirname(__file__), "fixtures/*.lock") diff --git a/test/test_db.py b/test/test_db.py index de093cbd8..72d63b44b 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -3,12 +3,13 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import os.path as osp + from git.db import GitCmdObjectDB from git.exc import BadObject -from test.lib import TestBase from git.util import bin_to_hex -import os.path as osp +from test.lib import TestBase class TestDB(TestBase): diff --git a/test/test_diff.py b/test/test_diff.py index 96fbc60e3..928a9f428 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -14,6 +14,7 @@ from git import NULL_TREE, Diff, DiffIndex, Diffable, GitCommandError, Repo, Submodule from git.cmd import Git + from test.lib import StringProcessAdapter, TestBase, fixture, with_rw_directory diff --git a/test/test_docs.py b/test/test_docs.py index 409f66bb3..b3547c1de 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -5,6 +5,7 @@ import gc import os +import os.path import sys import pytest @@ -12,8 +13,6 @@ from test.lib import TestBase from test.lib.helper import with_rw_directory -import os.path - class Tutorials(TestBase): def tearDown(self): diff --git a/test/test_exc.py b/test/test_exc.py index 3f4d0b803..c1eae7240 100644 --- a/test/test_exc.py +++ b/test/test_exc.py @@ -3,9 +3,11 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +from itertools import product import re import ddt + from git.exc import ( InvalidGitRepositoryError, WorkTreeRepositoryUnsupported, @@ -20,9 +22,8 @@ RepositoryDirtyError, ) from git.util import remove_password_if_present -from test.lib import TestBase -import itertools as itt +from test.lib import TestBase _cmd_argvs = ( @@ -79,7 +80,7 @@ def test_ExceptionsHaveBaseClass(self): for ex_class in exception_classes: self.assertTrue(issubclass(ex_class, GitError)) - @ddt.data(*list(itt.product(_cmd_argvs, _causes_n_substrings, _streams_n_substrings))) + @ddt.data(*list(product(_cmd_argvs, _causes_n_substrings, _streams_n_substrings))) def test_CommandError_unicode(self, case): argv, (cause, subs), stream = case cls = CommandError diff --git a/test/test_fun.py b/test/test_fun.py index 2d30d355a..b8593b400 100644 --- a/test/test_fun.py +++ b/test/test_fun.py @@ -2,27 +2,26 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from io import BytesIO -from stat import S_IFDIR, S_IFREG, S_IFLNK, S_IXUSR +from stat import S_IFDIR, S_IFLNK, S_IFREG, S_IXUSR from os import stat import os.path as osp +from gitdb.base import IStream +from gitdb.typ import str_tree_type + from git import Git from git.index import IndexFile -from git.index.fun import ( - aggressive_tree_merge, - stat_mode_to_index_mode, -) +from git.index.fun import aggressive_tree_merge, stat_mode_to_index_mode from git.objects.fun import ( traverse_tree_recursive, traverse_trees_recursive, - tree_to_stream, tree_entries_from_data, + tree_to_stream, ) from git.repo.fun import find_worktree_git_dir -from test.lib import TestBase, with_rw_repo, with_rw_directory from git.util import bin_to_hex, cygpath, join_path_native -from gitdb.base import IStream -from gitdb.typ import str_tree_type + +from test.lib import TestBase, with_rw_directory, with_rw_repo class TestFun(TestBase): diff --git a/test/test_git.py b/test/test_git.py index dae0f6a39..112e2f0eb 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -25,8 +25,9 @@ import ddt -from git import Git, refresh, GitCommandError, GitCommandNotFound, Repo, cmd +from git import Git, GitCommandError, GitCommandNotFound, Repo, cmd, refresh from git.util import cwd, finalize_process + from test.lib import TestBase, fixture_path, with_rw_directory diff --git a/test/test_index.py b/test/test_index.py index 622e7ca9a..b92258c92 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -17,18 +17,12 @@ import sys import tempfile +from gitdb.base import IStream + import ddt import pytest -from git import ( - BlobFilter, - Diff, - Git, - IndexFile, - Object, - Repo, - Tree, -) +from git import BlobFilter, Diff, Git, IndexFile, Object, Repo, Tree from git.exc import ( CheckoutError, GitCommandError, @@ -41,7 +35,7 @@ from git.index.util import TemporaryFileSwap from git.objects import Blob from git.util import Actor, cwd, hex_to_bin, rmtree -from gitdb.base import IStream + from test.lib import ( TestBase, VirtualEnvironment, diff --git a/test/test_reflog.py b/test/test_reflog.py index 1bd2e5dab..7ce64219a 100644 --- a/test/test_reflog.py +++ b/test/test_reflog.py @@ -5,9 +5,10 @@ import tempfile from git.objects import IndexObject -from git.refs import RefLogEntry, RefLog +from git.refs import RefLog, RefLogEntry +from git.util import Actor, hex_to_bin, rmtree + from test.lib import TestBase, fixture_path -from git.util import Actor, rmtree, hex_to_bin class TestRefLog(TestBase): diff --git a/test/test_refs.py b/test/test_refs.py index 28db70c6e..08096e69e 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -4,27 +4,28 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ from itertools import chain +import os.path as osp from pathlib import Path +import tempfile + +from gitdb.exc import BadName from git import ( - Reference, - Head, - TagReference, - RemoteReference, Commit, - SymbolicReference, GitCommandError, - RefLog, GitConfigParser, + Head, + RefLog, + Reference, + RemoteReference, + SymbolicReference, + TagReference, ) from git.objects.tag import TagObject -from test.lib import TestBase, with_rw_repo +import git.refs as refs from git.util import Actor -from gitdb.exc import BadName -import git.refs as refs -import os.path as osp -import tempfile +from test.lib import TestBase, with_rw_repo class TestRefs(TestBase): diff --git a/test/test_remote.py b/test/test_remote.py index f84452deb..5ddb41bc0 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -28,7 +28,7 @@ ) from git.cmd import Git from git.exc import UnsafeOptionError, UnsafeProtocolError -from git.util import rmtree, HIDE_WINDOWS_FREEZE_ERRORS, IterableList +from git.util import HIDE_WINDOWS_FREEZE_ERRORS, IterableList, rmtree from test.lib import ( GIT_DAEMON_PORT, TestBase, @@ -37,7 +37,6 @@ with_rw_repo, ) - # Make sure we have repeatable results. random.seed(0) diff --git a/test/test_repo.py b/test/test_repo.py index 238f94712..e38da5bb6 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -36,13 +36,10 @@ Submodule, Tree, ) -from git.exc import ( - BadObject, - UnsafeOptionError, - UnsafeProtocolError, -) +from git.exc import BadObject, UnsafeOptionError, UnsafeProtocolError from git.repo.fun import touch from git.util import bin_to_hex, cwd, cygpath, join_path_native, rmfile, rmtree + from test.lib import TestBase, fixture, with_rw_directory, with_rw_repo diff --git a/test/test_stats.py b/test/test_stats.py index 4efb6f313..eec73c802 100644 --- a/test/test_stats.py +++ b/test/test_stats.py @@ -3,10 +3,11 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from test.lib import TestBase, fixture from git import Stats from git.compat import defenc +from test.lib import TestBase, fixture + class TestStats(TestBase): def test_list_from_string(self): diff --git a/test/test_submodule.py b/test/test_submodule.py index ee7795dbb..d88f9dab0 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -27,6 +27,7 @@ from git.objects.submodule.root import RootModule, RootUpdateProgress from git.repo.fun import find_submodule_git_dir, touch from git.util import HIDE_WINDOWS_KNOWN_ERRORS, join_path_native, to_native_path_linux + from test.lib import TestBase, with_rw_directory, with_rw_repo diff --git a/test/test_tree.py b/test/test_tree.py index 0c06b950c..73158113d 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -8,8 +8,9 @@ from pathlib import Path import subprocess -from git.objects import Tree, Blob +from git.objects import Blob, Tree from git.util import cwd + from test.lib import TestBase, with_rw_directory diff --git a/test/test_util.py b/test/test_util.py index 369896581..dad2f3dcd 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -38,6 +38,7 @@ remove_password_if_present, rmtree, ) + from test.lib import TestBase, with_rw_repo From d524c76a460b339b224c59b6753607eb2e546b2a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 19 Mar 2024 01:02:20 -0400 Subject: [PATCH 0889/1392] Fix slightly unsorted imports in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 143206653..f28fedb85 100755 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ #!/usr/bin/env python import os -import sys from pathlib import Path +import sys from typing import Sequence from setuptools import setup, find_packages From 838eb923751aad3063a47ef24c0b806e9c4e2453 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 19 Mar 2024 13:22:46 -0400 Subject: [PATCH 0890/1392] Clarify how tag objects are usually tree-ish and commit-ish This revises the docstrings of the Tree_ish and Commit_ish unions, to make clearer that tags objects are usually used to identify commits and are thus usually tree-ish and commit-ish. This change relates to the discussion in #1878, starting at: https://github.com/gitpython-developers/GitPython/pull/1878#pullrequestreview-1938343498 --- git/types.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/git/types.py b/git/types.py index 230422dff..a93ebdb4f 100644 --- a/git/types.py +++ b/git/types.py @@ -75,7 +75,7 @@ """ Tree_ish = Union["Commit", "Tree", "TagObject"] -"""Union of :class:`~git.objects.base.Object`-based types that are sometimes tree-ish. +"""Union of :class:`~git.objects.base.Object`-based types that are typically tree-ish. See :manpage:`gitglossary(7)` on "tree-ish": https://git-scm.com/docs/gitglossary#def_tree-ish @@ -83,10 +83,11 @@ :note: :class:`~git.objects.tree.Tree` and :class:`~git.objects.commit.Commit` are the classes whose instances are all tree-ish. This union includes them, but also - :class:`~git.objects.tag.TagObject`, only **some** of whose instances are tree-ish. + :class:`~git.objects.tag.TagObject`, only **most** of whose instances are tree-ish. Whether a particular :class:`~git.objects.tag.TagObject` peels (recursively dereferences) to a tree or commit, rather than a blob, can in general only be known - at runtime. + at runtime. In practice, git tag objects are nearly always used for tagging commits, + and such tags are tree-ish because commits are tree-ish. :note: See also the :class:`AnyGitObject` union of all four classes corresponding to git @@ -94,7 +95,7 @@ """ Commit_ish = Union["Commit", "TagObject"] -"""Union of :class:`~git.objects.base.Object`-based types that are sometimes commit-ish. +"""Union of :class:`~git.objects.base.Object`-based types that are typically commit-ish. See :manpage:`gitglossary(7)` on "commit-ish": https://git-scm.com/docs/gitglossary#def_commit-ish @@ -102,10 +103,11 @@ :note: :class:`~git.objects.commit.Commit` is the only class whose instances are all commit-ish. This union type includes :class:`~git.objects.commit.Commit`, but also - :class:`~git.objects.tag.TagObject`, only **some** of whose instances are + :class:`~git.objects.tag.TagObject`, only **most** of whose instances are commit-ish. Whether a particular :class:`~git.objects.tag.TagObject` peels (recursively dereferences) to a commit, rather than a tree or blob, can in general - only be known at runtime. + only be known at runtime. In practice, git tag objects are nearly always used for + tagging commits, and such tags are of course commit-ish. :note: See also the :class:`AnyGitObject` union of all four classes corresponding to git From 2382891377f60c1467c1965b25e3ecf293f39b80 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 19:41:55 -0400 Subject: [PATCH 0891/1392] Test that deprecated Diff.renamed property warns This starts on a test.deprecation subpackage for deprecation tests. Having these tests in a separate directory inside the test suite may or may not be how they will ultimately be orgnaized, but it has two advantages: - Once all the tests are written, it should be easy to see what in GitPython is deprecated. - Some deprecation warnings -- those on module or class attribute access -- will require the introduction of new dynamic behavior, and thus run the risk of breaking static type checking. So that should be checked for, where applicable. But currently the test suite has no type annotations and is not checked by mypy. Having deprecation-related tests under the same path will make it easier to enable mypy for just this part of the test suite (for now). It is also for this latter reason that the one test so far is written without using the GitPython test suite's existing fixtures whose uses are harder to annotate. This may be changed if warranted, though some of the more complex deprecation-related tests may benefit from being written as pure pytest tests. Although a number of deprecated features in GitPython do already issue warnings, Diff.renamed is one of the features that does not yet do so. So the newly introduced test will fail until that is fixed in the next commit. --- test/deprecation/__init__.py | 2 ++ test/deprecation/test_various.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 test/deprecation/__init__.py create mode 100644 test/deprecation/test_various.py diff --git a/test/deprecation/__init__.py b/test/deprecation/__init__.py new file mode 100644 index 000000000..56b5d89db --- /dev/null +++ b/test/deprecation/__init__.py @@ -0,0 +1,2 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ diff --git a/test/deprecation/test_various.py b/test/deprecation/test_various.py new file mode 100644 index 000000000..9f948bc48 --- /dev/null +++ b/test/deprecation/test_various.py @@ -0,0 +1,20 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Tests of assorted deprecation warnings with no extra subtleties to check.""" + +from git.diff import NULL_TREE +from git.repo import Repo + +import pytest + + +def test_diff_renamed_warns(tmp_path): + (tmp_path / "a.txt").write_text("hello\n", encoding="utf-8") + repo = Repo.init(tmp_path) + repo.index.add(["a.txt"]) + commit = repo.index.commit("Initial commit") + (diff,) = commit.diff(NULL_TREE) # Exactly one file in the diff. + + with pytest.deprecated_call(): + diff.renamed From e7dec7d0eecc362f02418820a146735a68430fbd Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 19:52:17 -0400 Subject: [PATCH 0892/1392] Have the deprecated Diff.renamed property issue a warning --- git/diff.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/git/diff.py b/git/diff.py index 0e39fe7a8..f89b12d98 100644 --- a/git/diff.py +++ b/git/diff.py @@ -7,6 +7,7 @@ import enum import re +import warnings from git.cmd import handle_process_output from git.compat import defenc @@ -554,6 +555,11 @@ def renamed(self) -> bool: This property is deprecated. Please use the :attr:`renamed_file` property instead. """ + warnings.warn( + "Diff.renamed is deprecated, use Diff.renamed_file instead", + DeprecationWarning, + stacklevel=2, + ) return self.renamed_file @property From a8f109ca1704827b017349d6d97f8a0a3229d9a8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 20:04:25 -0400 Subject: [PATCH 0893/1392] Fix exception in Popen.__del__ in test on Windows The newly introduced test would pass, but show: Exception ignored in: Traceback (most recent call last): File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.12_3.12.752.0_x64__qbz5n2kfra8p0\Lib\subprocess.py", line 1130, in __del__ self._internal_poll(_deadstate=_maxsize) File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.12_3.12.752.0_x64__qbz5n2kfra8p0\Lib\subprocess.py", line 1575, in _internal_poll if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ OSError: [WinError 6] The handle is invalid This was due to how, at least for now, the deprecation warning tests are not using GitPython's normal fixtures, which help clean things up on Windows. This adds a small amount of ad-hoc cleanup. It also moves the acquisition of the Diff object into a pytest fixture, which can be reused for the immediately forthcoming test that the preferred property does not warn. --- test/deprecation/test_various.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/deprecation/test_various.py b/test/deprecation/test_various.py index 9f948bc48..056e0ac0d 100644 --- a/test/deprecation/test_various.py +++ b/test/deprecation/test_various.py @@ -3,18 +3,27 @@ """Tests of assorted deprecation warnings with no extra subtleties to check.""" -from git.diff import NULL_TREE -from git.repo import Repo +import gc import pytest +from git.diff import NULL_TREE +from git.repo import Repo + -def test_diff_renamed_warns(tmp_path): +@pytest.fixture +def single_diff(tmp_path): + """Fixture to supply a single-file diff.""" (tmp_path / "a.txt").write_text("hello\n", encoding="utf-8") repo = Repo.init(tmp_path) repo.index.add(["a.txt"]) commit = repo.index.commit("Initial commit") (diff,) = commit.diff(NULL_TREE) # Exactly one file in the diff. + yield diff + del repo, commit, diff + gc.collect() + +def test_diff_renamed_warns(single_diff): with pytest.deprecated_call(): - diff.renamed + single_diff.renamed From fffa6cea663b179c0e5d53cf5eb93159e2bbc3b0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 20:31:10 -0400 Subject: [PATCH 0894/1392] Test that the preferred renamed_file property does not warn --- test/deprecation/test_various.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/deprecation/test_various.py b/test/deprecation/test_various.py index 056e0ac0d..9dd95f723 100644 --- a/test/deprecation/test_various.py +++ b/test/deprecation/test_various.py @@ -4,6 +4,7 @@ """Tests of assorted deprecation warnings with no extra subtleties to check.""" import gc +import warnings import pytest @@ -25,5 +26,14 @@ def single_diff(tmp_path): def test_diff_renamed_warns(single_diff): + """The deprecated Diff.renamed property issues a deprecation warning.""" with pytest.deprecated_call(): single_diff.renamed + + +def test_diff_renamed_file_does_not_warn(single_diff): + """The preferred Diff.renamed_file property issues no deprecation warning.""" + with warnings.catch_warnings(): + # FIXME: Refine this to filter for deprecation warnings from GitPython. + warnings.simplefilter("error", DeprecationWarning) + single_diff.renamed_file From bc111b799b06990fd7dde6385265d0e6d3a6289d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 20:31:30 -0400 Subject: [PATCH 0895/1392] Add a TODO for simplifying the single_diff fixture --- test/deprecation/test_various.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/deprecation/test_various.py b/test/deprecation/test_various.py index 9dd95f723..efdb0a57c 100644 --- a/test/deprecation/test_various.py +++ b/test/deprecation/test_various.py @@ -15,6 +15,7 @@ @pytest.fixture def single_diff(tmp_path): """Fixture to supply a single-file diff.""" + # TODO: Consider making a fake diff rather than using a real repo and commit. (tmp_path / "a.txt").write_text("hello\n", encoding="utf-8") repo = Repo.init(tmp_path) repo.index.add(["a.txt"]) From e3728c3dca9cac2b49f827dbf5d1a100602f33d9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 20:54:50 -0400 Subject: [PATCH 0896/1392] Decompose new fixture logic better This will be even more helpful when testing a deprecated member of the Commit class (not yet done). --- test/deprecation/test_various.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/test/deprecation/test_various.py b/test/deprecation/test_various.py index efdb0a57c..398367b61 100644 --- a/test/deprecation/test_various.py +++ b/test/deprecation/test_various.py @@ -13,28 +13,33 @@ @pytest.fixture -def single_diff(tmp_path): - """Fixture to supply a single-file diff.""" - # TODO: Consider making a fake diff rather than using a real repo and commit. +def commit(tmp_path): + """Fixture to supply a one-commit repo's commit, enough for deprecation tests.""" (tmp_path / "a.txt").write_text("hello\n", encoding="utf-8") repo = Repo.init(tmp_path) repo.index.add(["a.txt"]) - commit = repo.index.commit("Initial commit") + yield repo.index.commit("Initial commit") + del repo + gc.collect() + + +@pytest.fixture +def diff(commit): + """Fixture to supply a single-file diff.""" + # TODO: Consider making a fake diff rather than using a real repo and commit. (diff,) = commit.diff(NULL_TREE) # Exactly one file in the diff. yield diff - del repo, commit, diff - gc.collect() -def test_diff_renamed_warns(single_diff): +def test_diff_renamed_warns(diff): """The deprecated Diff.renamed property issues a deprecation warning.""" with pytest.deprecated_call(): - single_diff.renamed + diff.renamed -def test_diff_renamed_file_does_not_warn(single_diff): +def test_diff_renamed_file_does_not_warn(diff): """The preferred Diff.renamed_file property issues no deprecation warning.""" with warnings.catch_warnings(): # FIXME: Refine this to filter for deprecation warnings from GitPython. warnings.simplefilter("error", DeprecationWarning) - single_diff.renamed_file + diff.renamed_file From ff4b58dd56d382b26d83f2f41f7d11d15e9db8dc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 20:57:40 -0400 Subject: [PATCH 0897/1392] Extract no-deprecation-warning asserter as a context manager + Remove the TODO suggesting the diff not be computed from a real commit, since we're going to have the logic for that in use in forthcoming commit-specific deprecation warning tests anyway. --- test/deprecation/test_various.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/test/deprecation/test_various.py b/test/deprecation/test_various.py index 398367b61..60f94311f 100644 --- a/test/deprecation/test_various.py +++ b/test/deprecation/test_various.py @@ -3,6 +3,7 @@ """Tests of assorted deprecation warnings with no extra subtleties to check.""" +import contextlib import gc import warnings @@ -12,6 +13,15 @@ from git.repo import Repo +@contextlib.contextmanager +def _assert_no_deprecation_warning(): + """Context manager to assert that code does not issue any deprecation warnings.""" + with warnings.catch_warnings(): + # FIXME: Refine this to filter for deprecation warnings from GitPython. + warnings.simplefilter("error", DeprecationWarning) + yield + + @pytest.fixture def commit(tmp_path): """Fixture to supply a one-commit repo's commit, enough for deprecation tests.""" @@ -26,7 +36,6 @@ def commit(tmp_path): @pytest.fixture def diff(commit): """Fixture to supply a single-file diff.""" - # TODO: Consider making a fake diff rather than using a real repo and commit. (diff,) = commit.diff(NULL_TREE) # Exactly one file in the diff. yield diff @@ -39,7 +48,5 @@ def test_diff_renamed_warns(diff): def test_diff_renamed_file_does_not_warn(diff): """The preferred Diff.renamed_file property issues no deprecation warning.""" - with warnings.catch_warnings(): - # FIXME: Refine this to filter for deprecation warnings from GitPython. - warnings.simplefilter("error", DeprecationWarning) + with _assert_no_deprecation_warning(): diff.renamed_file From 2c526962ebf0a5d4d31a725577db112f2ce60447 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 21:04:06 -0400 Subject: [PATCH 0898/1392] Test that the deprecated Commit.trailers property warns And that the non-deprecated recommended alternative trailers_list and trailers_dict properties do not warn. The test that trailers warns does not yet pass yet, because it has not yet been made to warn. --- test/deprecation/test_various.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/deprecation/test_various.py b/test/deprecation/test_various.py index 60f94311f..99ce882b0 100644 --- a/test/deprecation/test_various.py +++ b/test/deprecation/test_various.py @@ -50,3 +50,21 @@ def test_diff_renamed_file_does_not_warn(diff): """The preferred Diff.renamed_file property issues no deprecation warning.""" with _assert_no_deprecation_warning(): diff.renamed_file + + +def test_commit_trailers_warns(commit): + """The deprecated Commit.trailers property issues a deprecation warning.""" + with pytest.deprecated_call(): + commit.trailers + + +def test_commit_trailers_list_does_not_warn(commit): + """The nondeprecated Commit.trailers_list property issues no deprecation warning.""" + with _assert_no_deprecation_warning(): + commit.trailers_list + + +def test_commit_trailers_dict_does_not_warn(commit): + """The nondeprecated Commit.trailers_dict property issues no deprecation warning.""" + with _assert_no_deprecation_warning(): + commit.trailers_dict From 03464d90a66caf6a5d8dbbd37318758331c85168 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 21:12:41 -0400 Subject: [PATCH 0899/1392] Have the deprecated Commit.trailers property issue a warning --- git/objects/commit.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/git/objects/commit.py b/git/objects/commit.py index 8de52980c..d957c9051 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -14,6 +14,7 @@ from subprocess import Popen, PIPE import sys from time import altzone, daylight, localtime, time, timezone +import warnings from gitdb import IStream @@ -399,6 +400,11 @@ def trailers(self) -> Dict[str, str]: Dictionary containing whitespace stripped trailer information. Only contains the latest instance of each trailer key. """ + warnings.warn( + "Commit.trailers is deprecated, use Commit.trailers_list or Commit.trailers_dict instead", + DeprecationWarning, + stacklevel=2, + ) return {k: v[0] for k, v in self.trailers_dict.items()} @property From 9d096e08d7645bc5206c4728b45efcf26486b635 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 21:51:11 -0400 Subject: [PATCH 0900/1392] Test that Traversable.{list_,}traverse, but not overrides, warn --- test/deprecation/test_various.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/deprecation/test_various.py b/test/deprecation/test_various.py index 99ce882b0..be3195a5e 100644 --- a/test/deprecation/test_various.py +++ b/test/deprecation/test_various.py @@ -10,6 +10,7 @@ import pytest from git.diff import NULL_TREE +from git.objects.util import Traversable from git.repo import Repo @@ -68,3 +69,27 @@ def test_commit_trailers_dict_does_not_warn(commit): """The nondeprecated Commit.trailers_dict property issues no deprecation warning.""" with _assert_no_deprecation_warning(): commit.trailers_dict + + +def test_traverse_list_traverse_in_base_class_warns(commit): + """Traversable.list_traverse's base implementation issues a deprecation warning.""" + with pytest.deprecated_call(): + Traversable.list_traverse(commit) + + +def test_traversable_list_traverse_override_does_not_warn(commit): + """Calling list_traverse on concrete subclasses is not deprecated, does not warn.""" + with _assert_no_deprecation_warning(): + commit.list_traverse() + + +def test_traverse_traverse_in_base_class_warns(commit): + """Traversable.traverse's base implementation issues a deprecation warning.""" + with pytest.deprecated_call(): + Traversable.traverse(commit) + + +def test_traverse_traverse_override_does_not_warn(commit): + """Calling traverse on concrete subclasses is not deprecated, does not warn.""" + with _assert_no_deprecation_warning(): + commit.traverse() From 21c2b72b019627dba81b35085117b79660567abd Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Mar 2024 01:51:54 -0400 Subject: [PATCH 0901/1392] Use the :exc: Sphinx role for DeprecationWarning Python warnings are exceptions, to facilitate converting them to errors by raising them. Thus the more specific :exc: role applies and is a better fit than the more general :class: role. --- git/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index 5839f9720..8c1c26012 100644 --- a/git/util.py +++ b/git/util.py @@ -1285,7 +1285,7 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> IterableList[T_I class IterableClassWatcher(type): - """Metaclass that issues :class:`DeprecationWarning` when :class:`git.util.Iterable` + """Metaclass that issues :exc:`DeprecationWarning` when :class:`git.util.Iterable` is subclassed.""" def __init__(cls, name: str, bases: Tuple, clsdict: Dict) -> None: From ca385a59439006355de02ddab9012031e8577e41 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Mar 2024 02:00:43 -0400 Subject: [PATCH 0902/1392] Test that subclassing deprecated git.util.Iterable warns And that subclassing the strongly preferred git.util.Iterable class does not warn. --- test/deprecation/test_various.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/deprecation/test_various.py b/test/deprecation/test_various.py index be3195a5e..71f9cf940 100644 --- a/test/deprecation/test_various.py +++ b/test/deprecation/test_various.py @@ -12,6 +12,7 @@ from git.diff import NULL_TREE from git.objects.util import Traversable from git.repo import Repo +from git.util import Iterable as _Iterable, IterableObj @contextlib.contextmanager @@ -93,3 +94,19 @@ def test_traverse_traverse_override_does_not_warn(commit): """Calling traverse on concrete subclasses is not deprecated, does not warn.""" with _assert_no_deprecation_warning(): commit.traverse() + + +def test_iterable_inheriting_warns(): + """Subclassing the deprecated git.util.Iterable issues a deprecation warning.""" + with pytest.deprecated_call(): + + class Derived(_Iterable): + pass + + +def test_iterable_obj_inheriting_does_not_warn(): + """Subclassing git.util.IterableObj is not deprecated, does not warn.""" + with _assert_no_deprecation_warning(): + + class Derived(IterableObj): + pass From 8bbcb26ea6e798a10969570d24cd5e9c401feed7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Mar 2024 11:43:13 -0400 Subject: [PATCH 0903/1392] Call repo.close() instead of manually collecting The code this replaces in the `commit` pytest fixture seems not to be needed anymore, and could arguably be removed now with no further related changes. But whether it is needed depends on subtle and sometimes nondeterministic factors, and may also vary across Python versions. To be safe, this replaces it with a call to the Repo instance's close method, which is in effect more robust than what was being done before, as it calls clear_cache on the the Git object that the Repo object uses, does a gitdb/smmap collection, and on Windows calls gc.collect both before and after that collection. This may happen immediately anyway if the Repo object is not reachable from any cycle, since the reference count should go to zero after each of the deprecation warning tests (the fixture's lifetime is that of the test case), and Repo.close is called in Repo.__del__. But this makes it happen immediately even if there is a cycle. --- test/deprecation/test_various.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/deprecation/test_various.py b/test/deprecation/test_various.py index 71f9cf940..a82b989b7 100644 --- a/test/deprecation/test_various.py +++ b/test/deprecation/test_various.py @@ -4,7 +4,6 @@ """Tests of assorted deprecation warnings with no extra subtleties to check.""" import contextlib -import gc import warnings import pytest @@ -31,8 +30,7 @@ def commit(tmp_path): repo = Repo.init(tmp_path) repo.index.add(["a.txt"]) yield repo.index.commit("Initial commit") - del repo - gc.collect() + repo.close() @pytest.fixture From b8ce99031d3183a895f15e49fb7149a56a653065 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Mar 2024 19:39:00 -0400 Subject: [PATCH 0904/1392] Better name and document the basic deprecation test module The other deprecation test modules this refers to don't exist yet but will be introduced soon. --- .../deprecation/{test_various.py => test_basic.py} | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) rename test/deprecation/{test_various.py => test_basic.py} (80%) diff --git a/test/deprecation/test_various.py b/test/deprecation/test_basic.py similarity index 80% rename from test/deprecation/test_various.py rename to test/deprecation/test_basic.py index a82b989b7..459d79268 100644 --- a/test/deprecation/test_various.py +++ b/test/deprecation/test_basic.py @@ -1,7 +1,19 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Tests of assorted deprecation warnings with no extra subtleties to check.""" +"""Tests of assorted deprecation warnings when there are no extra subtleties to check. + +This tests deprecation warnings where all that needs be verified is that a deprecated +property, function, or class issues a DeprecationWarning when used and, if applicable, +that recommended alternatives do not issue the warning. + +This is in contrast to other modules within test.deprecation, which test warnings where +there is a risk of breaking other runtime behavior, or of breaking static type checking +or making it less useful, by introducing the warning or in plausible future changes to +how the warning is implemented. That happens when it is necessary to customize attribute +access on a module or class, in a way it was not customized before, to issue a warning. +It is inapplicable to the deprecations whose warnings are tested in this module. +""" import contextlib import warnings From 61273aa2084ada8c48d0f9e511556d3d72eec32c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Mar 2024 20:49:34 -0400 Subject: [PATCH 0905/1392] Annotate basic deprecation tests; have mypy scan it - Add type hints to test/deprecation/basic.py. As its module docstring (already) notes, that test module does not contain code where it is specifically important that it be type checked to verify important properties of the code under test. However, other test.deprecation.* modules will, and it is much more convenient to be able to scan the whole directory than the directory except for one file. (Less importantly, for the deprecation tests to be easily readable as a coherent whole, it makes sense that all, not just most, would have annotations.) - Configure mypy in pyproject.toml so it can be run without arguments (mypy errors when run that way unless configured), where the effect is to scan the git/ directory, as was commonly done before, as well as the test/deprecation/ directory. - Change the CI test workflow, as well as tox.ini, to run mypy with no arguments instead of passing `-p git`, so that it will scan both the git package as it had before and the test.deprecation package (due to the new pyproject.toml configuration). - Change the readme to recommend running it that way, too. --- .github/workflows/pythonpackage.yml | 2 +- README.md | 2 +- pyproject.toml | 1 + test/deprecation/test_basic.py | 40 +++++++++++++++++++---------- tox.ini | 2 +- 5 files changed, 30 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 7cee0cd64..4c918a92d 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -88,7 +88,7 @@ jobs: - name: Check types with mypy run: | - mypy --python-version=${{ matrix.python-version }} -p git + mypy --python-version=${{ matrix.python-version }} env: MYPY_FORCE_COLOR: "1" TERM: "xterm-256color" # For color: https://github.com/python/mypy/issues/13817 diff --git a/README.md b/README.md index 30af532db..9bedaaae7 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ This includes the linting and autoformatting done by Ruff, as well as some other To typecheck, run: ```sh -mypy -p git +mypy ``` #### CI (and tox) diff --git a/pyproject.toml b/pyproject.toml index 1dc1e6aed..5eac2be09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ testpaths = "test" # Space separated list of paths from root e.g test tests doc [tool.mypy] python_version = "3.8" +files = ["git/", "test/deprecation/"] disallow_untyped_defs = true no_implicit_optional = true warn_redundant_casts = true diff --git a/test/deprecation/test_basic.py b/test/deprecation/test_basic.py index 459d79268..8ee7e72b1 100644 --- a/test/deprecation/test_basic.py +++ b/test/deprecation/test_basic.py @@ -25,9 +25,21 @@ from git.repo import Repo from git.util import Iterable as _Iterable, IterableObj +# typing ----------------------------------------------------------------- + +from typing import Generator, TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + + from git.diff import Diff + from git.objects.commit import Commit + +# ------------------------------------------------------------------------ + @contextlib.contextmanager -def _assert_no_deprecation_warning(): +def _assert_no_deprecation_warning() -> Generator[None, None, None]: """Context manager to assert that code does not issue any deprecation warnings.""" with warnings.catch_warnings(): # FIXME: Refine this to filter for deprecation warnings from GitPython. @@ -36,7 +48,7 @@ def _assert_no_deprecation_warning(): @pytest.fixture -def commit(tmp_path): +def commit(tmp_path: "Path") -> Generator["Commit", None, None]: """Fixture to supply a one-commit repo's commit, enough for deprecation tests.""" (tmp_path / "a.txt").write_text("hello\n", encoding="utf-8") repo = Repo.init(tmp_path) @@ -46,67 +58,67 @@ def commit(tmp_path): @pytest.fixture -def diff(commit): +def diff(commit: "Commit") -> Generator["Diff", None, None]: """Fixture to supply a single-file diff.""" (diff,) = commit.diff(NULL_TREE) # Exactly one file in the diff. yield diff -def test_diff_renamed_warns(diff): +def test_diff_renamed_warns(diff: "Diff") -> None: """The deprecated Diff.renamed property issues a deprecation warning.""" with pytest.deprecated_call(): diff.renamed -def test_diff_renamed_file_does_not_warn(diff): +def test_diff_renamed_file_does_not_warn(diff: "Diff") -> None: """The preferred Diff.renamed_file property issues no deprecation warning.""" with _assert_no_deprecation_warning(): diff.renamed_file -def test_commit_trailers_warns(commit): +def test_commit_trailers_warns(commit: "Commit") -> None: """The deprecated Commit.trailers property issues a deprecation warning.""" with pytest.deprecated_call(): commit.trailers -def test_commit_trailers_list_does_not_warn(commit): +def test_commit_trailers_list_does_not_warn(commit: "Commit") -> None: """The nondeprecated Commit.trailers_list property issues no deprecation warning.""" with _assert_no_deprecation_warning(): commit.trailers_list -def test_commit_trailers_dict_does_not_warn(commit): +def test_commit_trailers_dict_does_not_warn(commit: "Commit") -> None: """The nondeprecated Commit.trailers_dict property issues no deprecation warning.""" with _assert_no_deprecation_warning(): commit.trailers_dict -def test_traverse_list_traverse_in_base_class_warns(commit): +def test_traverse_list_traverse_in_base_class_warns(commit: "Commit") -> None: """Traversable.list_traverse's base implementation issues a deprecation warning.""" with pytest.deprecated_call(): Traversable.list_traverse(commit) -def test_traversable_list_traverse_override_does_not_warn(commit): +def test_traversable_list_traverse_override_does_not_warn(commit: "Commit") -> None: """Calling list_traverse on concrete subclasses is not deprecated, does not warn.""" with _assert_no_deprecation_warning(): commit.list_traverse() -def test_traverse_traverse_in_base_class_warns(commit): +def test_traverse_traverse_in_base_class_warns(commit: "Commit") -> None: """Traversable.traverse's base implementation issues a deprecation warning.""" with pytest.deprecated_call(): Traversable.traverse(commit) -def test_traverse_traverse_override_does_not_warn(commit): +def test_traverse_traverse_override_does_not_warn(commit: "Commit") -> None: """Calling traverse on concrete subclasses is not deprecated, does not warn.""" with _assert_no_deprecation_warning(): commit.traverse() -def test_iterable_inheriting_warns(): +def test_iterable_inheriting_warns() -> None: """Subclassing the deprecated git.util.Iterable issues a deprecation warning.""" with pytest.deprecated_call(): @@ -114,7 +126,7 @@ class Derived(_Iterable): pass -def test_iterable_obj_inheriting_does_not_warn(): +def test_iterable_obj_inheriting_does_not_warn() -> None: """Subclassing git.util.IterableObj is not deprecated, does not warn.""" with _assert_no_deprecation_warning(): diff --git a/tox.ini b/tox.ini index 33074a78a..fc62fa587 100644 --- a/tox.ini +++ b/tox.ini @@ -30,7 +30,7 @@ description = Typecheck with mypy base_python = py{39,310,311,312,38,37} set_env = MYPY_FORCE_COLOR = 1 -commands = mypy -p git +commands = mypy ignore_outcome = true [testenv:html] From b7a3d8c08537d00aac065b7dcbe1a4896919ee07 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 19 Mar 2024 17:05:06 -0400 Subject: [PATCH 0906/1392] Start on top-level module attribute access regression tests --- test/deprecation/test_attributes.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 test/deprecation/test_attributes.py diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py new file mode 100644 index 000000000..2673cc172 --- /dev/null +++ b/test/deprecation/test_attributes.py @@ -0,0 +1,10 @@ +"""Tests for dynamic and static attribute errors.""" + +import pytest + +import git + + +def test_no_attribute() -> None: + with pytest.raises(AttributeError): + git.foo From 105f50056126a73e8cbfeb0d1157695fe6b296b1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 19 Mar 2024 17:15:02 -0400 Subject: [PATCH 0907/1392] Test attribute access and importing separately Rather than only testing attribute access. --- test/deprecation/test_attributes.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 2673cc172..aea3278f8 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -2,9 +2,14 @@ import pytest -import git +def test_cannot_get_undefined() -> None: + import git -def test_no_attribute() -> None: with pytest.raises(AttributeError): git.foo + + +def test_cannot_import_undefined() -> None: + with pytest.raises(ImportError): + from git import foo From 859e38cfb395e6a937c30fedd36a9b3896747188 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Mar 2024 20:14:22 -0400 Subject: [PATCH 0908/1392] Expand to test top-level deprecated names --- test/deprecation/test_attributes.py | 89 ++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index aea3278f8..428dab236 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -1,5 +1,7 @@ """Tests for dynamic and static attribute errors.""" +import importlib + import pytest @@ -12,4 +14,89 @@ def test_cannot_get_undefined() -> None: def test_cannot_import_undefined() -> None: with pytest.raises(ImportError): - from git import foo + from git import foo # noqa: F401 + + +def test_util_alias_access_resolves() -> None: + """These resolve for now, though they're private we do not guarantee this.""" + import git + + assert git.util is git.index.util + + +def test_util_alias_import_resolves() -> None: + from git import util + import git + + util is git.index.util + + +def test_util_alias_access_warns() -> None: + import git + + with pytest.deprecated_call() as ctx: + git.util + + assert len(ctx) == 1 + message = ctx[0].message.args[0] + assert "git.util" in message + assert "git.index.util" in message + assert "should not be relied on" in message + + +def test_util_alias_import_warns() -> None: + with pytest.deprecated_call() as ctx: + from git import util # noqa: F401 + + message = ctx[0].message.args[0] + assert "git.util" in message + assert "git.index.util" in message + assert "should not be relied on" in message + + +_parametrize_by_private_alias = pytest.mark.parametrize( + "name, fullname", + [ + ("head", "git.refs.head"), + ("log", "git.refs.log"), + ("reference", "git.refs.reference"), + ("symbolic", "git.refs.symbolic"), + ("tag", "git.refs.tag"), + ("base", "git.index.base"), + ("fun", "git.index.fun"), + ("typ", "git.index.typ"), + ], +) + + +@_parametrize_by_private_alias +def test_private_module_alias_access_resolves(name: str, fullname: str) -> None: + """These resolve for now, though they're private we do not guarantee this.""" + import git + + assert getattr(git, name) is importlib.import_module(fullname) + + +@_parametrize_by_private_alias +def test_private_module_alias_import_resolves(name: str, fullname: str) -> None: + exec(f"from git import {name}") + locals()[name] is importlib.import_module(fullname) + + +@_parametrize_by_private_alias +def test_private_module_alias_access_warns(name: str, fullname: str) -> None: + import git + + with pytest.deprecated_call() as ctx: + getattr(git, name) + + assert len(ctx) == 1 + assert ctx[0].message.args[0].endswith(f"Use {fullname} instead.") + + +@_parametrize_by_private_alias +def test_private_module_alias_import_warns(name: str, fullname: str) -> None: + with pytest.deprecated_call() as ctx: + exec(f"from git import {name}") + + assert ctx[0].message.args[0].endswith(f"Use {fullname} instead.") From 46a739da24331849323a7c583ffd61a51583d3c1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Mar 2024 20:20:04 -0400 Subject: [PATCH 0909/1392] Hoist `import git` to module level in test module Because it's going to be necessary to express things in terms of it in parametrization markings, in order for mypy to show the expected errors for names that are available dynamically but deliberately static type errors. --- test/deprecation/test_attributes.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 428dab236..85aa7a579 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -4,10 +4,10 @@ import pytest +import git -def test_cannot_get_undefined() -> None: - import git +def test_cannot_get_undefined() -> None: with pytest.raises(AttributeError): git.foo @@ -19,21 +19,16 @@ def test_cannot_import_undefined() -> None: def test_util_alias_access_resolves() -> None: """These resolve for now, though they're private we do not guarantee this.""" - import git - assert git.util is git.index.util def test_util_alias_import_resolves() -> None: from git import util - import git util is git.index.util def test_util_alias_access_warns() -> None: - import git - with pytest.deprecated_call() as ctx: git.util @@ -72,8 +67,6 @@ def test_util_alias_import_warns() -> None: @_parametrize_by_private_alias def test_private_module_alias_access_resolves(name: str, fullname: str) -> None: """These resolve for now, though they're private we do not guarantee this.""" - import git - assert getattr(git, name) is importlib.import_module(fullname) @@ -85,8 +78,6 @@ def test_private_module_alias_import_resolves(name: str, fullname: str) -> None: @_parametrize_by_private_alias def test_private_module_alias_access_warns(name: str, fullname: str) -> None: - import git - with pytest.deprecated_call() as ctx: getattr(git, name) From a2df3a8283274dda9236d0d41cf44a38317560cb Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Mar 2024 20:40:17 -0400 Subject: [PATCH 0910/1392] Test static typing of private module aliases This tests that mypy considers them not to be present. That mypy is configured with `warn_unused_ignores = true` is key, since that is what verifies that the type errors really do occur, based on the suppressions written for them. --- test/deprecation/test_attributes.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 85aa7a579..53612bde2 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -9,12 +9,12 @@ def test_cannot_get_undefined() -> None: with pytest.raises(AttributeError): - git.foo + git.foo # type: ignore[attr-defined] def test_cannot_import_undefined() -> None: with pytest.raises(ImportError): - from git import foo # noqa: F401 + from git import foo # type: ignore[attr-defined] # noqa: F401 def test_util_alias_access_resolves() -> None: @@ -49,6 +49,21 @@ def test_util_alias_import_warns() -> None: assert "should not be relied on" in message +def test_private_module_aliases() -> None: + """These exist dynamically but mypy will show them as absent (intentionally). + + More detailed dynamic behavior is examined in the subsequent test cases. + """ + git.head # type: ignore[attr-defined] + git.log # type: ignore[attr-defined] + git.reference # type: ignore[attr-defined] + git.symbolic # type: ignore[attr-defined] + git.tag # type: ignore[attr-defined] + git.base # type: ignore[attr-defined] + git.fun # type: ignore[attr-defined] + git.typ # type: ignore[attr-defined] + + _parametrize_by_private_alias = pytest.mark.parametrize( "name, fullname", [ From a15a830d47089ba625374fa3fd65e8568b7e0372 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Mar 2024 20:52:56 -0400 Subject: [PATCH 0911/1392] Improve a couple test case docstrings --- test/deprecation/test_attributes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 53612bde2..6df1359f5 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -18,7 +18,7 @@ def test_cannot_import_undefined() -> None: def test_util_alias_access_resolves() -> None: - """These resolve for now, though they're private we do not guarantee this.""" + """These resolve for now, though they're private and we do not guarantee this.""" assert git.util is git.index.util @@ -50,7 +50,7 @@ def test_util_alias_import_warns() -> None: def test_private_module_aliases() -> None: - """These exist dynamically but mypy will show them as absent (intentionally). + """These exist dynamically (for now) but mypy treats them as absent (intentionally). More detailed dynamic behavior is examined in the subsequent test cases. """ From dbaa535e47b61db0f9ce29c65b2a6616f6454196 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Mar 2024 20:57:05 -0400 Subject: [PATCH 0912/1392] Add a couple missing assert keywords --- test/deprecation/test_attributes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 6df1359f5..b9ca1d7c2 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -25,7 +25,7 @@ def test_util_alias_access_resolves() -> None: def test_util_alias_import_resolves() -> None: from git import util - util is git.index.util + assert util is git.index.util def test_util_alias_access_warns() -> None: @@ -88,7 +88,7 @@ def test_private_module_alias_access_resolves(name: str, fullname: str) -> None: @_parametrize_by_private_alias def test_private_module_alias_import_resolves(name: str, fullname: str) -> None: exec(f"from git import {name}") - locals()[name] is importlib.import_module(fullname) + assert locals()[name] is importlib.import_module(fullname) @_parametrize_by_private_alias From d00c843434806389bfb0bf7992505f358e97513f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Mar 2024 21:00:05 -0400 Subject: [PATCH 0913/1392] Clarify how test_private_module_aliases is statically checkable --- test/deprecation/test_attributes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index b9ca1d7c2..386ae1838 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -52,6 +52,9 @@ def test_util_alias_import_warns() -> None: def test_private_module_aliases() -> None: """These exist dynamically (for now) but mypy treats them as absent (intentionally). + This code verifies the effect of static type checking when analyzed by mypy, if mypy + is configured with ``warn_unused_ignores = true``. + More detailed dynamic behavior is examined in the subsequent test cases. """ git.head # type: ignore[attr-defined] From 983fda774a6eedda3b62ac2fa94ce54675a5f662 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 21 Mar 2024 02:04:21 -0400 Subject: [PATCH 0914/1392] Move mark-sharing tests into a class --- test/deprecation/test_attributes.py | 46 +++++++++++++---------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 386ae1838..7af77000f 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -49,8 +49,8 @@ def test_util_alias_import_warns() -> None: assert "should not be relied on" in message -def test_private_module_aliases() -> None: - """These exist dynamically (for now) but mypy treats them as absent (intentionally). +def test_private_module_aliases_exist_dynamically() -> None: + """These exist at runtime (for now) but mypy treats them as absent (intentionally). This code verifies the effect of static type checking when analyzed by mypy, if mypy is configured with ``warn_unused_ignores = true``. @@ -67,7 +67,7 @@ def test_private_module_aliases() -> None: git.typ # type: ignore[attr-defined] -_parametrize_by_private_alias = pytest.mark.parametrize( +@pytest.mark.parametrize( "name, fullname", [ ("head", "git.refs.head"), @@ -80,32 +80,26 @@ def test_private_module_aliases() -> None: ("typ", "git.index.typ"), ], ) +class TestPrivateModuleAliases: + """Tests of the private module aliases' shared specific runtime behaviors.""" + def test_private_module_alias_access_resolves(self, name: str, fullname: str) -> None: + """These resolve for now, though they're private we do not guarantee this.""" + assert getattr(git, name) is importlib.import_module(fullname) -@_parametrize_by_private_alias -def test_private_module_alias_access_resolves(name: str, fullname: str) -> None: - """These resolve for now, though they're private we do not guarantee this.""" - assert getattr(git, name) is importlib.import_module(fullname) - - -@_parametrize_by_private_alias -def test_private_module_alias_import_resolves(name: str, fullname: str) -> None: - exec(f"from git import {name}") - assert locals()[name] is importlib.import_module(fullname) - - -@_parametrize_by_private_alias -def test_private_module_alias_access_warns(name: str, fullname: str) -> None: - with pytest.deprecated_call() as ctx: - getattr(git, name) + def test_private_module_alias_import_resolves(self, name: str, fullname: str) -> None: + exec(f"from git import {name}") + assert locals()[name] is importlib.import_module(fullname) - assert len(ctx) == 1 - assert ctx[0].message.args[0].endswith(f"Use {fullname} instead.") + def test_private_module_alias_access_warns(self, name: str, fullname: str) -> None: + with pytest.deprecated_call() as ctx: + getattr(git, name) + assert len(ctx) == 1 + assert ctx[0].message.args[0].endswith(f"Use {fullname} instead.") -@_parametrize_by_private_alias -def test_private_module_alias_import_warns(name: str, fullname: str) -> None: - with pytest.deprecated_call() as ctx: - exec(f"from git import {name}") + def test_private_module_alias_import_warns(self, name: str, fullname: str) -> None: + with pytest.deprecated_call() as ctx: + exec(f"from git import {name}") - assert ctx[0].message.args[0].endswith(f"Use {fullname} instead.") + assert ctx[0].message.args[0].endswith(f"Use {fullname} instead.") From 19acd4cf551dfd6c7774e1ca7794ef83b321c8b6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Mar 2024 01:33:00 -0400 Subject: [PATCH 0915/1392] Add FIXME for what to do next --- test/deprecation/test_attributes.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 7af77000f..f249ae871 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -103,3 +103,14 @@ def test_private_module_alias_import_warns(self, name: str, fullname: str) -> No exec(f"from git import {name}") assert ctx[0].message.args[0].endswith(f"Use {fullname} instead.") + + +reveal_type(git.util.git_working_dir) + +# FIXME: Add one or more test cases that access something like git.util.git_working_dir +# to verify that it is available, and also use assert_type on it to ensure mypy knows +# that accesses to expressions of the form git.util.XYZ resolve to git.index.util.XYZ. +# +# It may be necessary for GitPython, in git/__init__.py, to import util from git.index +# explicitly before (still) deleting the util global, in order for mypy to know what is +# going on. Also check pyright. From f39bbb5172987b0462b5d1845c0cb0cf3824b3d5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Mar 2024 10:52:32 -0400 Subject: [PATCH 0916/1392] Fix a test docstring --- test/deprecation/test_attributes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index f249ae871..69a1aa1f3 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -84,7 +84,7 @@ class TestPrivateModuleAliases: """Tests of the private module aliases' shared specific runtime behaviors.""" def test_private_module_alias_access_resolves(self, name: str, fullname: str) -> None: - """These resolve for now, though they're private we do not guarantee this.""" + """These resolve for now, though they're private and we do not guarantee this.""" assert getattr(git, name) is importlib.import_module(fullname) def test_private_module_alias_import_resolves(self, name: str, fullname: str) -> None: From aee7078e28b5de9bb5cd605ff23f37d7328dccc9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Mar 2024 18:56:02 -0400 Subject: [PATCH 0917/1392] Test resolution into git.index.util using git.util --- test/deprecation/test_attributes.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 69a1aa1f3..74f51a09e 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -1,6 +1,7 @@ """Tests for dynamic and static attribute errors.""" import importlib +from typing import Type import pytest @@ -28,6 +29,23 @@ def test_util_alias_import_resolves() -> None: assert util is git.index.util +def test_util_alias_members_resolve() -> None: + """git.index.util members can be accessed via git.util, and mypy recognizes it.""" + # TODO: When typing_extensions is made a test dependency, use assert_type for this. + gu_tfs = git.util.TemporaryFileSwap + from git.index.util import TemporaryFileSwap + + def accepts_tfs_type(t: Type[TemporaryFileSwap]) -> None: + pass + + def rejects_tfs_type(t: Type[git.Git]) -> None: + pass + + accepts_tfs_type(gu_tfs) + rejects_tfs_type(gu_tfs) # type: ignore[arg-type] + assert gu_tfs is TemporaryFileSwap + + def test_util_alias_access_warns() -> None: with pytest.deprecated_call() as ctx: git.util @@ -103,14 +121,3 @@ def test_private_module_alias_import_warns(self, name: str, fullname: str) -> No exec(f"from git import {name}") assert ctx[0].message.args[0].endswith(f"Use {fullname} instead.") - - -reveal_type(git.util.git_working_dir) - -# FIXME: Add one or more test cases that access something like git.util.git_working_dir -# to verify that it is available, and also use assert_type on it to ensure mypy knows -# that accesses to expressions of the form git.util.XYZ resolve to git.index.util.XYZ. -# -# It may be necessary for GitPython, in git/__init__.py, to import util from git.index -# explicitly before (still) deleting the util global, in order for mypy to know what is -# going on. Also check pyright. From 7f4a19135c755065538d13ed4faeb9c10cc203c8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Mar 2024 19:01:11 -0400 Subject: [PATCH 0918/1392] Fix brittle way of checking warning messages Which was causing a type error. --- test/deprecation/test_attributes.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 74f51a09e..0f142fbe7 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -51,7 +51,7 @@ def test_util_alias_access_warns() -> None: git.util assert len(ctx) == 1 - message = ctx[0].message.args[0] + message = str(ctx[0].message) assert "git.util" in message assert "git.index.util" in message assert "should not be relied on" in message @@ -61,7 +61,7 @@ def test_util_alias_import_warns() -> None: with pytest.deprecated_call() as ctx: from git import util # noqa: F401 - message = ctx[0].message.args[0] + message = str(ctx[0].message) assert "git.util" in message assert "git.index.util" in message assert "should not be relied on" in message @@ -114,10 +114,12 @@ def test_private_module_alias_access_warns(self, name: str, fullname: str) -> No getattr(git, name) assert len(ctx) == 1 - assert ctx[0].message.args[0].endswith(f"Use {fullname} instead.") + message = str(ctx[0].message) + assert message.endswith(f"Use {fullname} instead.") def test_private_module_alias_import_warns(self, name: str, fullname: str) -> None: with pytest.deprecated_call() as ctx: exec(f"from git import {name}") - assert ctx[0].message.args[0].endswith(f"Use {fullname} instead.") + message = str(ctx[0].message) + assert message.endswith(f"Use {fullname} instead.") From d08a5768f6e8aa78de5f10c7e7a0777d2e4dfec3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 22 Mar 2024 19:09:44 -0400 Subject: [PATCH 0919/1392] Clarify todo --- test/deprecation/test_attributes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 0f142fbe7..829ff29d2 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -31,7 +31,6 @@ def test_util_alias_import_resolves() -> None: def test_util_alias_members_resolve() -> None: """git.index.util members can be accessed via git.util, and mypy recognizes it.""" - # TODO: When typing_extensions is made a test dependency, use assert_type for this. gu_tfs = git.util.TemporaryFileSwap from git.index.util import TemporaryFileSwap @@ -41,8 +40,10 @@ def accepts_tfs_type(t: Type[TemporaryFileSwap]) -> None: def rejects_tfs_type(t: Type[git.Git]) -> None: pass + # TODO: When typing_extensions is made a test dependency, use assert_type for this. accepts_tfs_type(gu_tfs) rejects_tfs_type(gu_tfs) # type: ignore[arg-type] + assert gu_tfs is TemporaryFileSwap From 9d58e6d367dc4651213e674b9f586e0a18d787bc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 14:35:36 -0400 Subject: [PATCH 0920/1392] Start reorganizing new tests more in the GitPython style --- test/deprecation/test_attributes.py | 142 +++++++++++++++------------- 1 file changed, 77 insertions(+), 65 deletions(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 829ff29d2..97aa9bc50 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -1,4 +1,8 @@ -"""Tests for dynamic and static attribute errors.""" +"""Tests for dynamic and static attribute errors in GitPython's top-level git module. + +Provided mypy has ``warn_unused_ignores = true`` set, running mypy on these test cases +checks static typing of the code under test. (Running pytest checks dynamic behavior.) +""" import importlib from typing import Type @@ -18,17 +22,6 @@ def test_cannot_import_undefined() -> None: from git import foo # type: ignore[attr-defined] # noqa: F401 -def test_util_alias_access_resolves() -> None: - """These resolve for now, though they're private and we do not guarantee this.""" - assert git.util is git.index.util - - -def test_util_alias_import_resolves() -> None: - from git import util - - assert util is git.index.util - - def test_util_alias_members_resolve() -> None: """git.index.util members can be accessed via git.util, and mypy recognizes it.""" gu_tfs = git.util.TemporaryFileSwap @@ -68,59 +61,78 @@ def test_util_alias_import_warns() -> None: assert "should not be relied on" in message -def test_private_module_aliases_exist_dynamically() -> None: - """These exist at runtime (for now) but mypy treats them as absent (intentionally). - - This code verifies the effect of static type checking when analyzed by mypy, if mypy - is configured with ``warn_unused_ignores = true``. - - More detailed dynamic behavior is examined in the subsequent test cases. - """ - git.head # type: ignore[attr-defined] - git.log # type: ignore[attr-defined] - git.reference # type: ignore[attr-defined] - git.symbolic # type: ignore[attr-defined] - git.tag # type: ignore[attr-defined] - git.base # type: ignore[attr-defined] - git.fun # type: ignore[attr-defined] - git.typ # type: ignore[attr-defined] - - -@pytest.mark.parametrize( - "name, fullname", - [ - ("head", "git.refs.head"), - ("log", "git.refs.log"), - ("reference", "git.refs.reference"), - ("symbolic", "git.refs.symbolic"), - ("tag", "git.refs.tag"), - ("base", "git.index.base"), - ("fun", "git.index.fun"), - ("typ", "git.index.typ"), - ], +# Split out util and have all its tests be separate, above. +_MODULE_ALIAS_TARGETS = ( + git.refs.head, + git.refs.log, + git.refs.reference, + git.refs.symbolic, + git.refs.tag, + git.index.base, + git.index.fun, + git.index.typ, + git.index.util, ) -class TestPrivateModuleAliases: - """Tests of the private module aliases' shared specific runtime behaviors.""" - def test_private_module_alias_access_resolves(self, name: str, fullname: str) -> None: - """These resolve for now, though they're private and we do not guarantee this.""" - assert getattr(git, name) is importlib.import_module(fullname) - def test_private_module_alias_import_resolves(self, name: str, fullname: str) -> None: - exec(f"from git import {name}") - assert locals()[name] is importlib.import_module(fullname) - - def test_private_module_alias_access_warns(self, name: str, fullname: str) -> None: - with pytest.deprecated_call() as ctx: - getattr(git, name) - - assert len(ctx) == 1 - message = str(ctx[0].message) - assert message.endswith(f"Use {fullname} instead.") - - def test_private_module_alias_import_warns(self, name: str, fullname: str) -> None: - with pytest.deprecated_call() as ctx: - exec(f"from git import {name}") - - message = str(ctx[0].message) - assert message.endswith(f"Use {fullname} instead.") +def test_private_module_alias_access_on_git_module() -> None: + """Private alias access works, warns, and except for util is a mypy error.""" + with pytest.deprecated_call() as ctx: + assert ( + git.head, # type: ignore[attr-defined] + git.log, # type: ignore[attr-defined] + git.reference, # type: ignore[attr-defined] + git.symbolic, # type: ignore[attr-defined] + git.tag, # type: ignore[attr-defined] + git.base, # type: ignore[attr-defined] + git.fun, # type: ignore[attr-defined] + git.typ, # type: ignore[attr-defined] + git.util, + ) == _MODULE_ALIAS_TARGETS + + messages = [str(w.message) for w in ctx] + for target, message in zip(_MODULE_ALIAS_TARGETS[:-1], messages[:-1], strict=True): + assert message.endswith(f"Use {target.__name__} instead.") + + util_message = messages[-1] + assert "git.util" in util_message + assert "git.index.util" in util_message + assert "should not be relied on" in util_message + + +def test_private_module_alias_import_from_git_module() -> None: + """Private alias import works, warns, and except for util is a mypy error.""" + with pytest.deprecated_call() as ctx: + from git import head # type: ignore[attr-defined] + from git import log # type: ignore[attr-defined] + from git import reference # type: ignore[attr-defined] + from git import symbolic # type: ignore[attr-defined] + from git import tag # type: ignore[attr-defined] + from git import base # type: ignore[attr-defined] + from git import fun # type: ignore[attr-defined] + from git import typ # type: ignore[attr-defined] + from git import util + + assert ( + head, + log, + reference, + symbolic, + tag, + base, + fun, + typ, + util, + ) == _MODULE_ALIAS_TARGETS + + # FIXME: This fails because, with imports, multiple consecutive accesses may occur. + # In practice, with CPython, it is always exactly two accesses, the first from the + # equivalent of a hasattr, and the second to fetch the attribute intentionally. + messages = [str(w.message) for w in ctx] + for target, message in zip(_MODULE_ALIAS_TARGETS[:-1], messages[:-1], strict=True): + assert message.endswith(f"Use {target.__name__} instead.") + + util_message = messages[-1] + assert "git.util" in util_message + assert "git.index.util" in util_message + assert "should not be relied on" in util_message From 45c128bcd82cd278d7926425179503c22ce1271c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 15:50:39 -0400 Subject: [PATCH 0921/1392] Finish reorganizing; fix assertion for duplicated messages To support the changes, this adds typing-extensions as a test dependency, since we are now importing from it in a test module. But this should probably be required and used conditionally based on whether the Python version has assert_type in its typing module. --- test-requirements.txt | 1 + test/deprecation/test_attributes.py | 98 ++++++++++++++--------------- 2 files changed, 50 insertions(+), 49 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index e1f5e2ed4..106b46aee 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,3 +8,4 @@ pytest-cov pytest-instafail pytest-mock pytest-sugar +typing-extensions diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 97aa9bc50..35e2e48bb 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -4,61 +4,72 @@ checks static typing of the code under test. (Running pytest checks dynamic behavior.) """ -import importlib +from itertools import groupby from typing import Type import pytest +from typing_extensions import assert_type import git -def test_cannot_get_undefined() -> None: +def test_cannot_access_undefined() -> None: + """Accessing a bogus attribute in git remains both a dynamic and static error.""" with pytest.raises(AttributeError): git.foo # type: ignore[attr-defined] def test_cannot_import_undefined() -> None: + """Importing a bogus attribute from git remains both a dynamic and static error.""" with pytest.raises(ImportError): from git import foo # type: ignore[attr-defined] # noqa: F401 -def test_util_alias_members_resolve() -> None: - """git.index.util members can be accessed via git.util, and mypy recognizes it.""" - gu_tfs = git.util.TemporaryFileSwap - from git.index.util import TemporaryFileSwap +def test_util_alias_access() -> None: + """Accessing util in git works, warns, and mypy verifies it and its attributes.""" + # The attribute access should succeed. + with pytest.deprecated_call() as ctx: + util = git.util - def accepts_tfs_type(t: Type[TemporaryFileSwap]) -> None: - pass + # There should be exactly one warning and it should have our util-specific message. + (message,) = [str(entry.message) for entry in ctx] + assert "git.util" in message + assert "git.index.util" in message + assert "should not be relied on" in message - def rejects_tfs_type(t: Type[git.Git]) -> None: - pass + # We check access through the util alias to the TemporaryFileSwap member, since it + # is slightly simpler to validate and reason about than the other public members, + # which are functions (specifically, higher-order functions for use as decorators). + from git.index.util import TemporaryFileSwap - # TODO: When typing_extensions is made a test dependency, use assert_type for this. - accepts_tfs_type(gu_tfs) - rejects_tfs_type(gu_tfs) # type: ignore[arg-type] + assert_type(util.TemporaryFileSwap, Type[TemporaryFileSwap]) - assert gu_tfs is TemporaryFileSwap + # This comes after the static assertion, just in case it would affect the inference. + assert util.TemporaryFileSwap is TemporaryFileSwap -def test_util_alias_access_warns() -> None: +def test_util_alias_import() -> None: + """Importing util from git works, warns, and mypy verifies it and its attributes.""" + # The import should succeed. with pytest.deprecated_call() as ctx: - git.util + from git import util - assert len(ctx) == 1 - message = str(ctx[0].message) + # There may be multiple warnings. In CPython there will be currently always be + # exactly two, possibly due to the equivalent of calling hasattr to do a pre-check + # prior to retrieving the attribute for actual use. However, all warnings should + # have the same message, and it should be our util-specific message. + (message,) = {str(entry.message) for entry in ctx} assert "git.util" in message assert "git.index.util" in message assert "should not be relied on" in message + # As above, we check access through the util alias to the TemporaryFileSwap member. + from git.index.util import TemporaryFileSwap -def test_util_alias_import_warns() -> None: - with pytest.deprecated_call() as ctx: - from git import util # noqa: F401 + assert_type(util.TemporaryFileSwap, Type[TemporaryFileSwap]) - message = str(ctx[0].message) - assert "git.util" in message - assert "git.index.util" in message - assert "should not be relied on" in message + # This comes after the static assertion, just in case it would affect the inference. + assert util.TemporaryFileSwap is TemporaryFileSwap # Split out util and have all its tests be separate, above. @@ -71,12 +82,11 @@ def test_util_alias_import_warns() -> None: git.index.base, git.index.fun, git.index.typ, - git.index.util, ) -def test_private_module_alias_access_on_git_module() -> None: - """Private alias access works, warns, and except for util is a mypy error.""" +def test_private_module_alias_access() -> None: + """Non-util private alias access works, warns, but is a deliberate mypy error.""" with pytest.deprecated_call() as ctx: assert ( git.head, # type: ignore[attr-defined] @@ -87,21 +97,16 @@ def test_private_module_alias_access_on_git_module() -> None: git.base, # type: ignore[attr-defined] git.fun, # type: ignore[attr-defined] git.typ, # type: ignore[attr-defined] - git.util, ) == _MODULE_ALIAS_TARGETS + # Each should have warned exactly once, and note what to use instead. messages = [str(w.message) for w in ctx] - for target, message in zip(_MODULE_ALIAS_TARGETS[:-1], messages[:-1], strict=True): + for target, message in zip(_MODULE_ALIAS_TARGETS, messages, strict=True): assert message.endswith(f"Use {target.__name__} instead.") - util_message = messages[-1] - assert "git.util" in util_message - assert "git.index.util" in util_message - assert "should not be relied on" in util_message - -def test_private_module_alias_import_from_git_module() -> None: - """Private alias import works, warns, and except for util is a mypy error.""" +def test_private_module_alias_import() -> None: + """Non-util private alias access works, warns, but is a deliberate mypy error.""" with pytest.deprecated_call() as ctx: from git import head # type: ignore[attr-defined] from git import log # type: ignore[attr-defined] @@ -111,7 +116,6 @@ def test_private_module_alias_import_from_git_module() -> None: from git import base # type: ignore[attr-defined] from git import fun # type: ignore[attr-defined] from git import typ # type: ignore[attr-defined] - from git import util assert ( head, @@ -122,17 +126,13 @@ def test_private_module_alias_import_from_git_module() -> None: base, fun, typ, - util, ) == _MODULE_ALIAS_TARGETS - # FIXME: This fails because, with imports, multiple consecutive accesses may occur. - # In practice, with CPython, it is always exactly two accesses, the first from the - # equivalent of a hasattr, and the second to fetch the attribute intentionally. - messages = [str(w.message) for w in ctx] - for target, message in zip(_MODULE_ALIAS_TARGETS[:-1], messages[:-1], strict=True): + # Each import may warn multiple times. In CPython there will be currently always be + # exactly two warnings per import, possibly due to the equivalent of calling hasattr + # to do a pre-check prior to retrieving the attribute for actual use. However, for + # each import, all messages should be the same and should note what to use instead. + messages_with_duplicates = [str(w.message) for w in ctx] + messages = [message for message, _ in groupby(messages_with_duplicates)] + for target, message in zip(_MODULE_ALIAS_TARGETS, messages, strict=True): assert message.endswith(f"Use {target.__name__} instead.") - - util_message = messages[-1] - assert "git.util" in util_message - assert "git.index.util" in util_message - assert "should not be relied on" in util_message From 247dc15fd81ecc806be732d7f1ef0c12e26920d8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 15:55:57 -0400 Subject: [PATCH 0922/1392] Add imports so pyright recognizes refs and index pyright still reports git.util as private, as it should. (mypy does not, or does not by default, report private member access. GitPython does not generally use pyright as part of development at this time, but I am checking some code with it during the process of writing the new tests.) --- test/deprecation/test_attributes.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 35e2e48bb..95feaaf8e 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -11,6 +11,14 @@ from typing_extensions import assert_type import git +import git.index.base +import git.index.fun +import git.index.typ +import git.refs.head +import git.refs.log +import git.refs.reference +import git.refs.symbolic +import git.refs.tag def test_cannot_access_undefined() -> None: From b05963c3749494f633117316a493ab2b179e0069 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 16:05:53 -0400 Subject: [PATCH 0923/1392] Expand and clarify test module docstring About why there are so many separate mypy suppressions even when they could be consolidated into a smaller number in some places. --- test/deprecation/test_attributes.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 95feaaf8e..218b9dd13 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -1,7 +1,11 @@ """Tests for dynamic and static attribute errors in GitPython's top-level git module. Provided mypy has ``warn_unused_ignores = true`` set, running mypy on these test cases -checks static typing of the code under test. (Running pytest checks dynamic behavior.) +checks static typing of the code under test. This is the reason for the many separate +single-line attr-defined suppressions, so those should not be replaced with a smaller +number of more broadly scoped suppressions, even where it is feasible to do so. + +Running pytest checks dynamic behavior as usual. """ from itertools import groupby From 074bbc72a84db7605f3136dc9f4b4ad822a3b481 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 16:08:37 -0400 Subject: [PATCH 0924/1392] Tiny import tweak --- test/deprecation/test_attributes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 218b9dd13..1bcca44d5 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -8,7 +8,7 @@ Running pytest checks dynamic behavior as usual. """ -from itertools import groupby +import itertools from typing import Type import pytest @@ -145,6 +145,6 @@ def test_private_module_alias_import() -> None: # to do a pre-check prior to retrieving the attribute for actual use. However, for # each import, all messages should be the same and should note what to use instead. messages_with_duplicates = [str(w.message) for w in ctx] - messages = [message for message, _ in groupby(messages_with_duplicates)] + messages = [message for message, _ in itertools.groupby(messages_with_duplicates)] for target, message in zip(_MODULE_ALIAS_TARGETS, messages, strict=True): assert message.endswith(f"Use {target.__name__} instead.") From 18608e472535149e542cc30ff95755ed962c9156 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 16:12:59 -0400 Subject: [PATCH 0925/1392] Pick a better name for _MODULE_ALIAS_TARGETS And add a docstring to document it, mainly to clarify that util is intentionally omitted from that constant. --- test/deprecation/test_attributes.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 1bcca44d5..6e98a5e09 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -85,7 +85,7 @@ def test_util_alias_import() -> None: # Split out util and have all its tests be separate, above. -_MODULE_ALIAS_TARGETS = ( +_PRIVATE_MODULE_ALIAS_TARGETS = ( git.refs.head, git.refs.log, git.refs.reference, @@ -95,6 +95,7 @@ def test_util_alias_import() -> None: git.index.fun, git.index.typ, ) +"""Targets of private aliases in the git module to some modules, not including util.""" def test_private_module_alias_access() -> None: @@ -109,11 +110,11 @@ def test_private_module_alias_access() -> None: git.base, # type: ignore[attr-defined] git.fun, # type: ignore[attr-defined] git.typ, # type: ignore[attr-defined] - ) == _MODULE_ALIAS_TARGETS + ) == _PRIVATE_MODULE_ALIAS_TARGETS # Each should have warned exactly once, and note what to use instead. messages = [str(w.message) for w in ctx] - for target, message in zip(_MODULE_ALIAS_TARGETS, messages, strict=True): + for target, message in zip(_PRIVATE_MODULE_ALIAS_TARGETS, messages, strict=True): assert message.endswith(f"Use {target.__name__} instead.") @@ -138,7 +139,7 @@ def test_private_module_alias_import() -> None: base, fun, typ, - ) == _MODULE_ALIAS_TARGETS + ) == _PRIVATE_MODULE_ALIAS_TARGETS # Each import may warn multiple times. In CPython there will be currently always be # exactly two warnings per import, possibly due to the equivalent of calling hasattr @@ -146,5 +147,5 @@ def test_private_module_alias_import() -> None: # each import, all messages should be the same and should note what to use instead. messages_with_duplicates = [str(w.message) for w in ctx] messages = [message for message, _ in itertools.groupby(messages_with_duplicates)] - for target, message in zip(_MODULE_ALIAS_TARGETS, messages, strict=True): + for target, message in zip(_PRIVATE_MODULE_ALIAS_TARGETS, messages, strict=True): assert message.endswith(f"Use {target.__name__} instead.") From 1f290f17943be338bb79a54ce1bef21e90da4402 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 16:36:22 -0400 Subject: [PATCH 0926/1392] Use typing_extensions only if needed This makes the typing-extensions test dependency < 3.11 only, and conditionally imports assert_type from typing or typing_extensions depending on the Python version. --- test-requirements.txt | 2 +- test/deprecation/test_attributes.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 106b46aee..75e9e81fa 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,4 +8,4 @@ pytest-cov pytest-instafail pytest-mock pytest-sugar -typing-extensions +typing-extensions ; python_version < "3.11" diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 6e98a5e09..2150186f9 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -9,8 +9,14 @@ """ import itertools +import sys from typing import Type +if sys.version_info >= (3, 11): + from typing import assert_type +else: + from typing_extensions import assert_type + import pytest from typing_extensions import assert_type From 7a4f7eb092fbf68b058de38ee2df193c86d4e34e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 17:07:32 -0400 Subject: [PATCH 0927/1392] Fix zip calls This omits strict=True, which is only supported in Python 3.10 and later, and instead explicitly asserts that the arguments are the same length (which is arguably better for its explicitness anyway). --- test/deprecation/test_attributes.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index 2150186f9..cd26f602b 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -120,7 +120,10 @@ def test_private_module_alias_access() -> None: # Each should have warned exactly once, and note what to use instead. messages = [str(w.message) for w in ctx] - for target, message in zip(_PRIVATE_MODULE_ALIAS_TARGETS, messages, strict=True): + + assert len(messages) == len(_PRIVATE_MODULE_ALIAS_TARGETS) + + for target, message in zip(_PRIVATE_MODULE_ALIAS_TARGETS, messages): assert message.endswith(f"Use {target.__name__} instead.") @@ -153,5 +156,8 @@ def test_private_module_alias_import() -> None: # each import, all messages should be the same and should note what to use instead. messages_with_duplicates = [str(w.message) for w in ctx] messages = [message for message, _ in itertools.groupby(messages_with_duplicates)] - for target, message in zip(_PRIVATE_MODULE_ALIAS_TARGETS, messages, strict=True): + + assert len(messages) == len(_PRIVATE_MODULE_ALIAS_TARGETS) + + for target, message in zip(_PRIVATE_MODULE_ALIAS_TARGETS, messages): assert message.endswith(f"Use {target.__name__} instead.") From 5977a6ec9e1646ac94514428a0ad2363be8da2f9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 17:19:39 -0400 Subject: [PATCH 0928/1392] Fix (and improve wording) of docstrings --- test/deprecation/test_attributes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index cd26f602b..e4fb39975 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -105,7 +105,7 @@ def test_util_alias_import() -> None: def test_private_module_alias_access() -> None: - """Non-util private alias access works, warns, but is a deliberate mypy error.""" + """Non-util private alias access works but warns and is a deliberate mypy error.""" with pytest.deprecated_call() as ctx: assert ( git.head, # type: ignore[attr-defined] @@ -128,7 +128,7 @@ def test_private_module_alias_access() -> None: def test_private_module_alias_import() -> None: - """Non-util private alias access works, warns, but is a deliberate mypy error.""" + """Non-util private alias import works but warns and is a deliberate mypy error.""" with pytest.deprecated_call() as ctx: from git import head # type: ignore[attr-defined] from git import log # type: ignore[attr-defined] From 5b1fa580400c386932eb9f66c568e4e0090e2779 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 17:46:57 -0400 Subject: [PATCH 0929/1392] Remove extra import "from typing_extensions" --- test/deprecation/test_attributes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_attributes.py index e4fb39975..eb909b299 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_attributes.py @@ -18,7 +18,6 @@ from typing_extensions import assert_type import pytest -from typing_extensions import assert_type import git import git.index.base From a07be0e35118874f682fa2e2be3b793170bf4853 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 18:12:19 -0400 Subject: [PATCH 0930/1392] Start on test_compat And rename test_attributes to test_toplevel accordingly. --- test/deprecation/test_compat.py | 33 +++++++++++++++++++ .../{test_attributes.py => test_toplevel.py} | 6 ++-- 2 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 test/deprecation/test_compat.py rename test/deprecation/{test_attributes.py => test_toplevel.py} (95%) diff --git a/test/deprecation/test_compat.py b/test/deprecation/test_compat.py new file mode 100644 index 000000000..dd2f0b0c2 --- /dev/null +++ b/test/deprecation/test_compat.py @@ -0,0 +1,33 @@ +"""Tests for dynamic and static errors and warnings in GitPython's git.compat module. + +These tests verify that the is_ aliases are available, and are even listed in +the output of dir(), but issue warnings, and that bogus (misspelled or unrecognized) +attribute access is still an error both at runtime and with mypy. This is similar to +some of the tests in test_toplevel, but the situation being tested here is simpler +because it does not involve unintuitive module aliasing or import behavior. So this only +tests attribute access, not "from" imports (whose behavior can be intuitively inferred). +""" + +import os +import sys + +import pytest + +import git.compat + + +_MESSAGE_LEADER = "{} and other is_ aliases are deprecated." + + +def test_cannot_access_undefined() -> None: + """Accessing a bogus attribute in git.compat remains a dynamic and static error.""" + with pytest.raises(AttributeError): + git.compat.foo # type: ignore[attr-defined] + + +def test_is_win() -> None: + with pytest.deprecated_call() as ctx: + value = git.compat.is_win + (message,) = [str(entry.message) for entry in ctx] # Exactly one message. + assert message.startswith(_MESSAGE_LEADER.format("git.compat.is_win")) + assert value == (os.name == "nt") diff --git a/test/deprecation/test_attributes.py b/test/deprecation/test_toplevel.py similarity index 95% rename from test/deprecation/test_attributes.py rename to test/deprecation/test_toplevel.py index eb909b299..2a662127b 100644 --- a/test/deprecation/test_attributes.py +++ b/test/deprecation/test_toplevel.py @@ -1,4 +1,4 @@ -"""Tests for dynamic and static attribute errors in GitPython's top-level git module. +"""Tests for dynamic and static errors and warnings in GitPython's top-level git module. Provided mypy has ``warn_unused_ignores = true`` set, running mypy on these test cases checks static typing of the code under test. This is the reason for the many separate @@ -31,13 +31,13 @@ def test_cannot_access_undefined() -> None: - """Accessing a bogus attribute in git remains both a dynamic and static error.""" + """Accessing a bogus attribute in git remains a dynamic and static error.""" with pytest.raises(AttributeError): git.foo # type: ignore[attr-defined] def test_cannot_import_undefined() -> None: - """Importing a bogus attribute from git remains both a dynamic and static error.""" + """Importing a bogus attribute from git remains a dynamic and static error.""" with pytest.raises(ImportError): from git import foo # type: ignore[attr-defined] # noqa: F401 From d4917d0a326d935ee0d6728af3268e9ece8b09df Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 18:19:36 -0400 Subject: [PATCH 0931/1392] Expand to test all three is_ aliases --- test/deprecation/test_compat.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/test/deprecation/test_compat.py b/test/deprecation/test_compat.py index dd2f0b0c2..6d2d87a39 100644 --- a/test/deprecation/test_compat.py +++ b/test/deprecation/test_compat.py @@ -25,9 +25,29 @@ def test_cannot_access_undefined() -> None: git.compat.foo # type: ignore[attr-defined] -def test_is_win() -> None: +def test_is_platform() -> None: + """The is_ aliases work, warn, and mypy accepts code accessing them.""" + fully_qualified_names = [ + "git.compat.is_win", + "git.compat.is_posix", + "git.compat.is_darwin", + ] + with pytest.deprecated_call() as ctx: - value = git.compat.is_win - (message,) = [str(entry.message) for entry in ctx] # Exactly one message. - assert message.startswith(_MESSAGE_LEADER.format("git.compat.is_win")) - assert value == (os.name == "nt") + is_win = git.compat.is_win + is_posix = git.compat.is_posix + is_darwin = git.compat.is_darwin + + messages = [str(entry.message) for entry in ctx] + assert len(messages) == 3 + + for fullname, message in zip(fully_qualified_names, messages): + assert message.startswith(_MESSAGE_LEADER.format(fullname)) + + # These exactly reproduce the expressions in the code under test, so they are not + # good for testing that the values are correct. Instead, the purpose of this test is + # to ensure that any dynamic machinery put in place in git.compat to cause warnings + # to be issued does not get in the way of the intended values being accessed. + assert is_win == (os.name == "nt") + assert is_posix == (os.name == "posix") + assert is_darwin == (sys.platform == "darwin") From f4e5f423f019c7d04798c896118f7fd48b8c3155 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 18:22:30 -0400 Subject: [PATCH 0932/1392] Slightly improve docstrings --- test/deprecation/test_compat.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/deprecation/test_compat.py b/test/deprecation/test_compat.py index 6d2d87a39..45d631e37 100644 --- a/test/deprecation/test_compat.py +++ b/test/deprecation/test_compat.py @@ -1,7 +1,7 @@ """Tests for dynamic and static errors and warnings in GitPython's git.compat module. -These tests verify that the is_ aliases are available, and are even listed in -the output of dir(), but issue warnings, and that bogus (misspelled or unrecognized) +These tests verify that the is_ attributes are available, and are even listed +in the output of dir(), but issue warnings, and that bogus (misspelled or unrecognized) attribute access is still an error both at runtime and with mypy. This is similar to some of the tests in test_toplevel, but the situation being tested here is simpler because it does not involve unintuitive module aliasing or import behavior. So this only @@ -15,8 +15,8 @@ import git.compat - _MESSAGE_LEADER = "{} and other is_ aliases are deprecated." +"""Form taken by the beginning of the warnings issues for is_ access.""" def test_cannot_access_undefined() -> None: @@ -26,7 +26,7 @@ def test_cannot_access_undefined() -> None: def test_is_platform() -> None: - """The is_ aliases work, warn, and mypy accepts code accessing them.""" + """The is_ attributes work, warn, and mypy accepts code accessing them.""" fully_qualified_names = [ "git.compat.is_win", "git.compat.is_posix", From d54f851d52f4217e904cbd163032aabbf33ec394 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 18:27:55 -0400 Subject: [PATCH 0933/1392] Add test of dir() on git.compat --- test/deprecation/test_compat.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/deprecation/test_compat.py b/test/deprecation/test_compat.py index 45d631e37..08911d17c 100644 --- a/test/deprecation/test_compat.py +++ b/test/deprecation/test_compat.py @@ -51,3 +51,17 @@ def test_is_platform() -> None: assert is_win == (os.name == "nt") assert is_posix == (os.name == "posix") assert is_darwin == (sys.platform == "darwin") + + +def test_dir() -> None: + """dir() on git.compat lists attributes meant to be public, even if deprecated.""" + expected = { + "defenc", + "safe_decode", + "safe_encode", + "win_encode", + "is_darwin", + "is_win", + "is_posix", + } + assert expected <= set(dir(git.compat)) From aaf046aba4b46fbe0dddf8c10d31f42b6c4d7c57 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 18:28:05 -0400 Subject: [PATCH 0934/1392] Add static type assertions to is_platform test --- test/deprecation/test_compat.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/deprecation/test_compat.py b/test/deprecation/test_compat.py index 08911d17c..c3cc5b0dd 100644 --- a/test/deprecation/test_compat.py +++ b/test/deprecation/test_compat.py @@ -11,6 +11,11 @@ import os import sys +if sys.version_info >= (3, 11): + from typing import assert_type +else: + from typing_extensions import assert_type + import pytest import git.compat @@ -38,6 +43,10 @@ def test_is_platform() -> None: is_posix = git.compat.is_posix is_darwin = git.compat.is_darwin + assert_type(is_win, bool) + assert_type(is_posix, bool) + assert_type(is_darwin, bool) + messages = [str(entry.message) for entry in ctx] assert len(messages) == 3 From 84d734d5b88cb8a03ab723f92bf83593d53d030f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 18:29:23 -0400 Subject: [PATCH 0935/1392] Refactor test_compat.test_dir for clarity --- test/deprecation/test_compat.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/deprecation/test_compat.py b/test/deprecation/test_compat.py index c3cc5b0dd..6e42d0209 100644 --- a/test/deprecation/test_compat.py +++ b/test/deprecation/test_compat.py @@ -64,7 +64,7 @@ def test_is_platform() -> None: def test_dir() -> None: """dir() on git.compat lists attributes meant to be public, even if deprecated.""" - expected = { + expected_subset = { "defenc", "safe_decode", "safe_encode", @@ -73,4 +73,5 @@ def test_dir() -> None: "is_win", "is_posix", } - assert expected <= set(dir(git.compat)) + actual = set(dir(git.compat)) + assert expected_subset <= actual From 3a621b38ee98a1d1413a12fcb68aae17d102f396 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 18:45:31 -0400 Subject: [PATCH 0936/1392] Add top-level dir() tests --- test/deprecation/test_toplevel.py | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/test/deprecation/test_toplevel.py b/test/deprecation/test_toplevel.py index 2a662127b..f74f09457 100644 --- a/test/deprecation/test_toplevel.py +++ b/test/deprecation/test_toplevel.py @@ -160,3 +160,52 @@ def test_private_module_alias_import() -> None: for target, message in zip(_PRIVATE_MODULE_ALIAS_TARGETS, messages): assert message.endswith(f"Use {target.__name__} instead.") + + +def test_dir_contains_public_attributes() -> None: + """All public attributes of the git module are present when dir() is called on it. + + This is naturally the case, but some ways of adding dynamic attribute access + behavior can change it, especially if __dir__ is defined but care is not taken to + preserve the contents that should already be present. + + Note that dir() should usually automatically list non-public attributes if they are + actually "physically" present as well, so the approach taken here to test it should + not be reproduced if __dir__ is added (instead, a call to globals() could be used, + as its keys list the automatic values). + """ + expected_subset = set(git.__all__) + actual = set(dir(git)) + assert expected_subset <= actual + + +def test_dir_does_not_contain_util() -> None: + """The util attribute is absent from the dir() of git. + + Because this behavior is less confusing than including it, where its meaning would + be assumed by users examining the dir() for what is available. + """ + assert "util" not in dir(git) + + +def test_dir_does_not_contain_private_module_aliases() -> None: + """Names from inside index and refs only pretend to be there and are not in dir(). + + The reason for omitting these is not that they are private, since private members + are usually included in dir() when actually present. Instead, these are only sort + of even there, no longer being imported and only being resolved dynamically for the + time being. In addition, it would be confusing to list these because doing so would + obscure the module structure of GitPython. + """ + expected_absent = { + "head", + "log", + "reference", + "symbolic", + "tag", + "base", + "fun", + "typ", + } + actual = set(dir(git)) + assert not (expected_absent & actual), "They should be completely disjoint." From 05e0878aef38a5fb1cb3d5860b3649cf7c1d4fda Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 18:47:27 -0400 Subject: [PATCH 0937/1392] Remove old comment meant as todo (that was done) --- test/deprecation/test_toplevel.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/deprecation/test_toplevel.py b/test/deprecation/test_toplevel.py index f74f09457..fe7045d46 100644 --- a/test/deprecation/test_toplevel.py +++ b/test/deprecation/test_toplevel.py @@ -89,7 +89,6 @@ def test_util_alias_import() -> None: assert util.TemporaryFileSwap is TemporaryFileSwap -# Split out util and have all its tests be separate, above. _PRIVATE_MODULE_ALIAS_TARGETS = ( git.refs.head, git.refs.log, From 3fe2f15d218744496e4af77b6a7926791480adfe Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 19:01:24 -0400 Subject: [PATCH 0938/1392] Test that top-level aliases point to modules with normal __name__ --- test/deprecation/test_toplevel.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/deprecation/test_toplevel.py b/test/deprecation/test_toplevel.py index fe7045d46..135cc53cc 100644 --- a/test/deprecation/test_toplevel.py +++ b/test/deprecation/test_toplevel.py @@ -102,6 +102,26 @@ def test_util_alias_import() -> None: """Targets of private aliases in the git module to some modules, not including util.""" +_PRIVATE_MODULE_ALIAS_TARGET_NAMES = ( + "git.refs.head", + "git.refs.log", + "git.refs.reference", + "git.refs.symbolic", + "git.refs.tag", + "git.index.base", + "git.index.fun", + "git.index.typ", +) +"""Expected ``__name__`` attributes of targets of private aliases in the git module.""" + + +def test_alias_target_module_names_are_by_location() -> None: + """The aliases are weird, but their targets are normal, even in ``__name__``.""" + actual = [module.__name__ for module in _PRIVATE_MODULE_ALIAS_TARGETS] + expected = list(_PRIVATE_MODULE_ALIAS_TARGET_NAMES) + assert actual == expected + + def test_private_module_alias_access() -> None: """Non-util private alias access works but warns and is a deliberate mypy error.""" with pytest.deprecated_call() as ctx: From 246cc1703f69e8eba791915bb025945b03abc86b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 19:02:56 -0400 Subject: [PATCH 0939/1392] Use names directly on other tests The tests are written broadly (per the style elsewhere in this test suite), but narrowing the message-checking tests in this specific way has the further advantage that the logic of the code under test will be less reflected in the logic of the tests, so that bugs are less likely to be missed by being duplicated across code and tests. --- test/deprecation/test_toplevel.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/deprecation/test_toplevel.py b/test/deprecation/test_toplevel.py index 135cc53cc..16c41d4e8 100644 --- a/test/deprecation/test_toplevel.py +++ b/test/deprecation/test_toplevel.py @@ -141,8 +141,8 @@ def test_private_module_alias_access() -> None: assert len(messages) == len(_PRIVATE_MODULE_ALIAS_TARGETS) - for target, message in zip(_PRIVATE_MODULE_ALIAS_TARGETS, messages): - assert message.endswith(f"Use {target.__name__} instead.") + for fullname, message in zip(_PRIVATE_MODULE_ALIAS_TARGET_NAMES, messages): + assert message.endswith(f"Use {fullname} instead.") def test_private_module_alias_import() -> None: @@ -177,8 +177,8 @@ def test_private_module_alias_import() -> None: assert len(messages) == len(_PRIVATE_MODULE_ALIAS_TARGETS) - for target, message in zip(_PRIVATE_MODULE_ALIAS_TARGETS, messages): - assert message.endswith(f"Use {target.__name__} instead.") + for fullname, message in zip(_PRIVATE_MODULE_ALIAS_TARGET_NAMES, messages): + assert message.endswith(f"Use {fullname} instead.") def test_dir_contains_public_attributes() -> None: From d7b6b31f632593bf9e280fbb2be87dd4e16ef7c5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 19:07:35 -0400 Subject: [PATCH 0940/1392] Fix a small docstring typo --- test/deprecation/test_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/deprecation/test_compat.py b/test/deprecation/test_compat.py index 6e42d0209..0da5462b2 100644 --- a/test/deprecation/test_compat.py +++ b/test/deprecation/test_compat.py @@ -21,7 +21,7 @@ import git.compat _MESSAGE_LEADER = "{} and other is_ aliases are deprecated." -"""Form taken by the beginning of the warnings issues for is_ access.""" +"""Form taken by the beginning of the warnings issued for is_ access.""" def test_cannot_access_undefined() -> None: From 96089c82c0d8982935bbd3326ccfea36ce72e43b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 19:17:15 -0400 Subject: [PATCH 0941/1392] Improve description in test module docstrings --- test/deprecation/test_compat.py | 2 +- test/deprecation/test_toplevel.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/deprecation/test_compat.py b/test/deprecation/test_compat.py index 0da5462b2..5007fa1cc 100644 --- a/test/deprecation/test_compat.py +++ b/test/deprecation/test_compat.py @@ -1,4 +1,4 @@ -"""Tests for dynamic and static errors and warnings in GitPython's git.compat module. +"""Tests for dynamic and static characteristics of git.compat module attributes. These tests verify that the is_ attributes are available, and are even listed in the output of dir(), but issue warnings, and that bogus (misspelled or unrecognized) diff --git a/test/deprecation/test_toplevel.py b/test/deprecation/test_toplevel.py index 16c41d4e8..398938616 100644 --- a/test/deprecation/test_toplevel.py +++ b/test/deprecation/test_toplevel.py @@ -1,4 +1,4 @@ -"""Tests for dynamic and static errors and warnings in GitPython's top-level git module. +"""Tests for dynamic and static characteristics of top-level git module attributes. Provided mypy has ``warn_unused_ignores = true`` set, running mypy on these test cases checks static typing of the code under test. This is the reason for the many separate From a0ef53778d4ae665474ebd96bd25ebbb340a8a16 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 23:12:12 -0400 Subject: [PATCH 0942/1392] Start on test_types --- test/deprecation/test_types.py | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 test/deprecation/test_types.py diff --git a/test/deprecation/test_types.py b/test/deprecation/test_types.py new file mode 100644 index 000000000..a2ac45829 --- /dev/null +++ b/test/deprecation/test_types.py @@ -0,0 +1,39 @@ +"""Tests for dynamic and static characteristics of git.types module attributes.""" + +import sys + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +import pytest + +import git.types + + +def test_cannot_access_undefined() -> None: + """Accessing a bogus attribute in git.types remains a dynamic and static error.""" + with pytest.raises(AttributeError): + git.types.foo # type: ignore[attr-defined] + + +def test_lit_commit_ish() -> None: + """ """ + # It would be fine to test attribute access rather than a "from" import. But a + # "from" import is more likely to appear in actual usage, so it is used here. + with pytest.deprecated_call() as ctx: + from git.types import Lit_commit_ish + + # As noted in test_toplevel.test_util_alias_import, there may be multiple warnings, + # but all with the same message. + (message,) = {str(entry.message) for entry in ctx} + assert "Lit_commit_ish is deprecated." in message + assert 'Literal["commit", "tag", "blob", "tree"]' in message, "Has old definition." + assert 'Literal["commit", "tag"]' in message, "Has new definition." + assert "GitObjectTypeString" in message, "Has new type name for old definition." + + _: Lit_commit_ish = "commit" # type: ignore[valid-type] + + # It should be as documented (even though deliberately unusable in static checks). + assert Lit_commit_ish == Literal["commit", "tag"] From 52e7360cad2ebde77a1302b205eb7cf9182a75c2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 23:13:53 -0400 Subject: [PATCH 0943/1392] Explain substring assertions in test_toplevel --- test/deprecation/test_toplevel.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/deprecation/test_toplevel.py b/test/deprecation/test_toplevel.py index 398938616..54dc8e358 100644 --- a/test/deprecation/test_toplevel.py +++ b/test/deprecation/test_toplevel.py @@ -76,9 +76,9 @@ def test_util_alias_import() -> None: # prior to retrieving the attribute for actual use. However, all warnings should # have the same message, and it should be our util-specific message. (message,) = {str(entry.message) for entry in ctx} - assert "git.util" in message - assert "git.index.util" in message - assert "should not be relied on" in message + assert "git.util" in message, "Has alias." + assert "git.index.util" in message, "Has target." + assert "should not be relied on" in message, "Distinct from other messages." # As above, we check access through the util alias to the TemporaryFileSwap member. from git.index.util import TemporaryFileSwap From e3675a086fe6ddfb6f8e4050497d47a74e851ed7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 23:17:55 -0400 Subject: [PATCH 0944/1392] Expand Lit_commit_ish test name and write docstring --- test/deprecation/test_types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/deprecation/test_types.py b/test/deprecation/test_types.py index a2ac45829..419435964 100644 --- a/test/deprecation/test_types.py +++ b/test/deprecation/test_types.py @@ -18,8 +18,8 @@ def test_cannot_access_undefined() -> None: git.types.foo # type: ignore[attr-defined] -def test_lit_commit_ish() -> None: - """ """ +def test_can_access_lit_commit_ish_but_it_is_not_usable() -> None: + """Lit_commit_ish_can be accessed, but warns and is an invalid type annotation.""" # It would be fine to test attribute access rather than a "from" import. But a # "from" import is more likely to appear in actual usage, so it is used here. with pytest.deprecated_call() as ctx: From 4857ff08fc9ae0183b65a4b94bd0813bd08a74b4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 23:26:43 -0400 Subject: [PATCH 0945/1392] Clarify test_compat.test_dir --- test/deprecation/test_compat.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/deprecation/test_compat.py b/test/deprecation/test_compat.py index 5007fa1cc..6699a24d5 100644 --- a/test/deprecation/test_compat.py +++ b/test/deprecation/test_compat.py @@ -63,15 +63,19 @@ def test_is_platform() -> None: def test_dir() -> None: - """dir() on git.compat lists attributes meant to be public, even if deprecated.""" + """dir() on git.compat includes all public attributes, even if deprecated. + + As dir() usually does, it also has nonpublic attributes, which should also not be + removed by a custom __dir__ function, but those are less important to test. + """ expected_subset = { + "is_win", + "is_posix", + "is_darwin", "defenc", "safe_decode", "safe_encode", "win_encode", - "is_darwin", - "is_win", - "is_posix", } actual = set(dir(git.compat)) assert expected_subset <= actual From 488cc13a5b125be886bb361342b8e4709c7944ba Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 Mar 2024 23:32:11 -0400 Subject: [PATCH 0946/1392] Add test of dir() on git.types --- test/deprecation/test_types.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/deprecation/test_types.py b/test/deprecation/test_types.py index 419435964..cd53fa210 100644 --- a/test/deprecation/test_types.py +++ b/test/deprecation/test_types.py @@ -37,3 +37,30 @@ def test_can_access_lit_commit_ish_but_it_is_not_usable() -> None: # It should be as documented (even though deliberately unusable in static checks). assert Lit_commit_ish == Literal["commit", "tag"] + + +def test_dir() -> None: + """dir() on git.types includes public names, even ``Lit_commit_ish``. + + It also contains private names that we don't test. See test_compat.test_dir. + """ + expected_subset = { + "PathLike", + "TBD", + "AnyGitObject", + "Tree_ish", + "Commit_ish", + "GitObjectTypeString", + "Lit_commit_ish", + "Lit_config_levels", + "ConfigLevels_Tup", + "CallableProgress", + "assert_never", + "Files_TD", + "Total_TD", + "HSH_TD", + "Has_Repo", + "Has_id_attribute", + } + actual = set(dir(git.types)) + assert expected_subset <= actual From 19b3c0820b607558b3bc52d3d9f3841295f04ac5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 24 Mar 2024 01:31:33 -0400 Subject: [PATCH 0947/1392] Clarify comment about is_ value assertions --- test/deprecation/test_compat.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/deprecation/test_compat.py b/test/deprecation/test_compat.py index 6699a24d5..22f733083 100644 --- a/test/deprecation/test_compat.py +++ b/test/deprecation/test_compat.py @@ -53,10 +53,10 @@ def test_is_platform() -> None: for fullname, message in zip(fully_qualified_names, messages): assert message.startswith(_MESSAGE_LEADER.format(fullname)) - # These exactly reproduce the expressions in the code under test, so they are not - # good for testing that the values are correct. Instead, the purpose of this test is - # to ensure that any dynamic machinery put in place in git.compat to cause warnings - # to be issued does not get in the way of the intended values being accessed. + # These assertions exactly reproduce the expressions in the code under test, so they + # are not good for testing that the values are correct. Instead, their purpose is to + # ensure that any dynamic machinery put in place in git.compat to cause warnings to + # be issued does not get in the way of the intended values being accessed. assert is_win == (os.name == "nt") assert is_posix == (os.name == "posix") assert is_darwin == (sys.platform == "darwin") From 28bd4a3f6e562eadc1018ee7c4d561a3c4352fb5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 20 Mar 2024 14:31:58 -0400 Subject: [PATCH 0948/1392] Issue warnings for some deprecated attributes of modules --- git/__init__.py | 97 ++++++++++++++++++++++++++++++++++++------------- git/compat.py | 40 ++++++++++++++++++-- git/types.py | 39 +++++++++++++++----- 3 files changed, 138 insertions(+), 38 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index a13030456..aa838d4f5 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -88,7 +88,10 @@ __version__ = "git" -from typing import List, Optional, Sequence, TYPE_CHECKING, Tuple, Union +from typing import Any, List, Optional, Sequence, TYPE_CHECKING, Tuple, Union + +if TYPE_CHECKING: + from types import ModuleType from gitdb.util import to_hex_sha @@ -144,11 +147,6 @@ SymbolicReference, Tag, TagReference, - head, # noqa: F401 # Nonpublic. May disappear! Use git.refs.head. - log, # noqa: F401 # Nonpublic. May disappear! Use git.refs.log. - reference, # noqa: F401 # Nonpublic. May disappear! Use git.refs.reference. - symbolic, # noqa: F401 # Nonpublic. May disappear! Use git.refs.symbolic. - tag, # noqa: F401 # Nonpublic. May disappear! Use git.refs.tag. ) from git.diff import ( # @NoMove INDEX, @@ -169,21 +167,6 @@ IndexEntry, IndexFile, StageType, - base, # noqa: F401 # Nonpublic. May disappear! Use git.index.base. - fun, # noqa: F401 # Nonpublic. May disappear! Use git.index.fun. - typ, # noqa: F401 # Nonpublic. May disappear! Use git.index.typ. - # - # NOTE: The expression `git.util` evaluates to git.index.util, and the import - # `from git import util` imports git.index.util, NOT git.util. It may not be - # feasible to change this until the next major version, to avoid breaking code - # inadvertently relying on it. If git.index.util really is what you want, use or - # import from that name, to avoid confusion. To use the "real" git.util module, - # write `from git.util import ...`, or access it as `sys.modules["git.util"]`. - # (This differs from other historical indirect-submodule imports that are - # unambiguously nonpublic and are subject to immediate removal. Here, the public - # git.util module, even though different, makes it less discoverable that the - # expression `git.util` refers to a non-public attribute of the git module.) - util, # noqa: F401 ) from git.util import ( # @NoMove Actor, @@ -196,7 +179,72 @@ except GitError as _exc: raise ImportError("%s: %s" % (_exc.__class__.__name__, _exc)) from _exc + +# NOTE: The expression `git.util` evaluates to git.index.util and `from git import util` +# imports git.index.util, NOT git.util. It may not be feasible to change this until the +# next major version, to avoid breaking code inadvertently relying on it. +# +# - If git.index.util *is* what you want, use or import from that, to avoid confusion. +# +# - To use the "real" git.util module, write `from git.util import ...`, or if necessary +# access it as `sys.modules["git.util"]`. +# +# (This differs from other indirect-submodule imports that are unambiguously non-public +# and subject to immediate removal. Here, the public git.util module, though different, +# makes less discoverable that the expression `git.util` refers to a non-public +# attribute of the git module.) +# +# This had come about by a wildcard import. Now that all intended imports are explicit, +# the intuitive but potentially incompatible binding occurs due to the usual rules for +# Python submodule bindings. So for now we delete that and let __getattr__ handle it. +# +del util # type: ignore[name-defined] # noqa: F821 + + +def _warned_import(message: str, fullname: str) -> "ModuleType": + import importlib + import warnings + + warnings.warn(message, DeprecationWarning, stacklevel=3) + return importlib.import_module(fullname) + + +def _getattr(name: str) -> Any: + # TODO: If __version__ is made dynamic and lazily fetched, put that case right here. + + if name == "util": + return _warned_import( + "The expression `git.util` and the import `from git import util` actually " + "reference git.index.util, and not the git.util module accessed in " + '`from git.util import XYZ` or `sys.modules["git.util"]`. This potentially ' + "confusing behavior is currently preserved for compatibility, but may be " + "changed in the future and should not be relied on.", + fullname="git.index.util", + ) + + for names, prefix in ( + ({"head", "log", "reference", "symbolic", "tag"}, "git.refs"), + ({"base", "fun", "typ"}, "git.index"), + ): + if name not in names: + continue + + fullname = f"{prefix}.{name}" + + return _warned_import( + f"{__name__}.{name} is a private alias of {fullname} and subject to " + f"immediate removal. Use {fullname} instead.", + fullname=fullname, + ) + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +if not TYPE_CHECKING: # Preserve static checking for undefined/misspelled attributes. + __getattr__ = _getattr + # { Initialize git executable path + GIT_OK = None @@ -232,12 +280,9 @@ def refresh(path: Optional[PathLike] = None) -> None: GIT_OK = True -# } END initialize git executable path - - -################# try: refresh() except Exception as _exc: raise ImportError("Failed to initialize: {0}".format(_exc)) from _exc -################# + +# } END initialize git executable path diff --git a/git/compat.py b/git/compat.py index 4ede8c985..f1b95a80e 100644 --- a/git/compat.py +++ b/git/compat.py @@ -23,7 +23,9 @@ AnyStr, Dict, # noqa: F401 IO, # noqa: F401 + List, Optional, + TYPE_CHECKING, Tuple, # noqa: F401 Type, # noqa: F401 Union, @@ -33,7 +35,39 @@ # --------------------------------------------------------------------------- -is_win = os.name == "nt" +_deprecated_platform_aliases = { + "is_win": os.name == "nt", + "is_posix": os.name == "posix", + "is_darwin": sys.platform == "darwin", +} + + +def _getattr(name: str) -> Any: + try: + value = _deprecated_platform_aliases[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + + import warnings + + warnings.warn( + f"{__name__}.{name} and other is_ aliases are deprecated. " + "Write the desired os.name or sys.platform check explicitly instead.", + DeprecationWarning, + stacklevel=2, + ) + return value + + +if not TYPE_CHECKING: # Preserve static checking for undefined/misspelled attributes. + __getattr__ = _getattr + + +def __dir__() -> List[str]: + return [*globals(), *_deprecated_platform_aliases] + + +is_win: bool """Deprecated alias for ``os.name == "nt"`` to check for native Windows. This is deprecated because it is clearer to write out :attr:`os.name` or @@ -45,7 +79,7 @@ Cygwin, use ``sys.platform == "cygwin"``. """ -is_posix = os.name == "posix" +is_posix: bool """Deprecated alias for ``os.name == "posix"`` to check for Unix-like ("POSIX") systems. This is deprecated because it clearer to write out :attr:`os.name` or @@ -58,7 +92,7 @@ (Darwin). """ -is_darwin = sys.platform == "darwin" +is_darwin: bool """Deprecated alias for ``sys.platform == "darwin"`` to check for macOS (Darwin). This is deprecated because it clearer to write out :attr:`os.name` or diff --git a/git/types.py b/git/types.py index a93ebdb4f..1be32de1d 100644 --- a/git/types.py +++ b/git/types.py @@ -7,11 +7,13 @@ Any, Callable, Dict, + List, NoReturn, Optional, Sequence as Sequence, Tuple, TYPE_CHECKING, + Type, TypeVar, Union, ) @@ -127,21 +129,40 @@ https://git-scm.com/docs/gitglossary#def_object_type """ -Lit_commit_ish = Literal["commit", "tag"] -"""Deprecated. Type of literal strings identifying sometimes-commitish git object types. +Lit_commit_ish: Type[Literal["commit", "tag"]] +"""Deprecated. Type of literal strings identifying typically-commitish git object types. Prior to a bugfix, this type had been defined more broadly. Any usage is in practice -ambiguous and likely to be incorrect. Instead of this type: +ambiguous and likely to be incorrect. This type has therefore been made a static type +error to appear in annotations. It is preserved, with a deprecated status, to avoid +introducing runtime errors in code that refers to it, but it should not be used. + +Instead of this type: * For the type of the string literals associated with :class:`Commit_ish`, use ``Literal["commit", "tag"]`` or create a new type alias for it. That is equivalent to - this type as currently defined. + this type as currently defined (but usable in statically checked type annotations). * For the type of all four string literals associated with :class:`AnyGitObject`, use :class:`GitObjectTypeString`. That is equivalent to the old definition of this type - prior to the bugfix. + prior to the bugfix (and is also usable in statically checked type annotations). """ + +def _getattr(name: str) -> Any: + if name == "Lit_commit_ish": + return Literal["commit", "tag"] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +if not TYPE_CHECKING: # Preserve static checking for undefined/misspelled attributes. + __getattr__ = _getattr + + +def __dir__() -> List[str]: + return [*globals(), "Lit_commit_ish"] + + # Config_levels --------------------------------------------------------- Lit_config_levels = Literal["system", "global", "user", "repository"] @@ -188,12 +209,12 @@ def assert_never(inp: NoReturn, raise_error: bool = True, exc: Union[Exception, :param inp: If all members are handled, the argument for `inp` will have the - :class:`~typing.Never`/:class:`~typing.NoReturn` type. Otherwise, the type will - mismatch and cause a mypy error. + :class:`~typing.Never`/:class:`~typing.NoReturn` type. + Otherwise, the type will mismatch and cause a mypy error. :param raise_error: - If ``True``, will also raise :exc:`ValueError` with a general "unhandled - literal" message, or the exception object passed as `exc`. + If ``True``, will also raise :exc:`ValueError` with a general + "unhandled literal" message, or the exception object passed as `exc`. :param exc: It not ``None``, this should be an already-constructed exception object, to be From dffa930a426747d9b30496fceb6efd621ab8795b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 24 Mar 2024 01:38:26 -0400 Subject: [PATCH 0949/1392] Refine deprecated module attributes and their warnings - In the top-level git module, import util from git.index so there is no ambiguity for tools in detecting which module the util attribute of the top-level git module (i.e., git.util in an *expression*) is, even though it's subsequently deleted (and then dynamically supplied when requested, in a way that is opaque to static type checkers due to being only when `not TYPE_CHECKING`). This seems to be necessary for some tools. Curiously, guarding the del statement under `not TYPE_CHECKING` seems *not* to be needed by any tools of any kind. It should still possibly be done, but that's not included in these changes. - Add the missing deprecation warning for git.types.Lit_commit_ish. - When importing the warnings module, do so with a top-level import as in the other GitPython modules that have long (and reasonably) done so. In git/__init__.py, there already had to be an import of importlib, which seemed like it should be done locally in case of delays. Neither the importlib module nor any of its submodules were already imported anywhere in GitPython, and the code that uses it will most often not be exercised. So there is a potential benefit to avoiding loading it when not needed. When writing a local import for that, I had included the warnings module as a local import as well. But this obscures the potential benefit of locally importing importlib, and could lead to ill-advised changes in the future based on the idea that the degree of justification to be local imports was the same for them both. --- git/__init__.py | 6 ++++-- git/compat.py | 3 +-- git/types.py | 17 ++++++++++++++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index aa838d4f5..fd1b87707 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -93,6 +93,8 @@ if TYPE_CHECKING: from types import ModuleType +import warnings + from gitdb.util import to_hex_sha from git.exc import ( @@ -167,6 +169,7 @@ IndexEntry, IndexFile, StageType, + util, ) from git.util import ( # @NoMove Actor, @@ -198,12 +201,11 @@ # the intuitive but potentially incompatible binding occurs due to the usual rules for # Python submodule bindings. So for now we delete that and let __getattr__ handle it. # -del util # type: ignore[name-defined] # noqa: F821 +del util def _warned_import(message: str, fullname: str) -> "ModuleType": import importlib - import warnings warnings.warn(message, DeprecationWarning, stacklevel=3) return importlib.import_module(fullname) diff --git a/git/compat.py b/git/compat.py index f1b95a80e..d7d9a55a9 100644 --- a/git/compat.py +++ b/git/compat.py @@ -13,6 +13,7 @@ import locale import os import sys +import warnings from gitdb.utils.encoding import force_bytes, force_text # noqa: F401 @@ -48,8 +49,6 @@ def _getattr(name: str) -> Any: except KeyError: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None - import warnings - warnings.warn( f"{__name__}.{name} and other is_ aliases are deprecated. " "Write the desired os.name or sys.platform check explicitly instead.", diff --git a/git/types.py b/git/types.py index 1be32de1d..584450146 100644 --- a/git/types.py +++ b/git/types.py @@ -17,6 +17,7 @@ TypeVar, Union, ) +import warnings if sys.version_info >= (3, 8): from typing import ( @@ -150,9 +151,19 @@ def _getattr(name: str) -> Any: - if name == "Lit_commit_ish": - return Literal["commit", "tag"] - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + if name != "Lit_commit_ish": + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + warnings.warn( + "Lit_commit_ish is deprecated. It is currently defined as " + '`Literal["commit", "tag"]`, which should be used in its place if desired. It ' + 'had previously been defined as `Literal["commit", "tag", "blob", "tree"]`, ' + "covering all four git object type strings including those that are never " + "commit-ish. For that, use the GitObjectTypeString type instead.", + DeprecationWarning, + stacklevel=2, + ) + return Literal["commit", "tag"] if not TYPE_CHECKING: # Preserve static checking for undefined/misspelled attributes. From 7ab27c5bb1891af9eec7a99e33ea35e234e322d5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 24 Mar 2024 19:31:00 -0400 Subject: [PATCH 0950/1392] Start on test module about Git.USE_SHELL and Git attributes --- test/deprecation/test_cmd_git.py | 168 +++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 test/deprecation/test_cmd_git.py diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py new file mode 100644 index 000000000..fc8bab6ee --- /dev/null +++ b/test/deprecation/test_cmd_git.py @@ -0,0 +1,168 @@ +"""Tests for static and dynamic characteristics of Git class and instance attributes. + +Currently this all relates to the deprecated :class:`Git.USE_SHELL` class attribute, +which can also be accessed through instances. Some tests directly verify its behavior, +including deprecation warnings, while others verify that other aspects of attribute +access are not inadvertently broken by mechanisms introduced to issue the warnings. +""" + +import contextlib +import sys +from typing import Generator + +if sys.version_info >= (3, 11): + from typing import assert_type +else: + from typing_extensions import assert_type + +import pytest + +from git.cmd import Git + +_USE_SHELL_DEPRECATED_FRAGMENT = "Git.USE_SHELL is deprecated" +"""Text contained in all USE_SHELL deprecation warnings, and starting most of them.""" + +_USE_SHELL_DANGEROUS_FRAGMENT = "Setting Git.USE_SHELL to True is unsafe and insecure" +"""Beginning text of USE_SHELL deprecation warnings when USE_SHELL is set True.""" + + +@pytest.fixture +def reset_backing_attribute() -> Generator[None, None, None]: + """Fixture to reset the private ``_USE_SHELL`` attribute. + + This is used to decrease the likelihood of state changes leaking out and affecting + other tests. But the goal is not to assert that ``_USE_SHELL`` is used, nor anything + about how or when it is used, which is an implementation detail subject to change. + + This is possible but inelegant to do with pytest's monkeypatch fixture, which only + restores attributes that it has previously been used to change, create, or remove. + """ + no_value = object() + try: + old_value = Git._USE_SHELL + except AttributeError: + old_value = no_value + + yield + + if old_value is no_value: + with contextlib.suppress(AttributeError): + del Git._USE_SHELL + else: + Git._USE_SHELL = old_value + + +def test_cannot_access_undefined_on_git_class() -> None: + """Accessing a bogus attribute on the Git class remains a dynamic and static error. + + This differs from Git instances, where most attribute names will dynamically + synthesize a "bound method" that runs a git subcommand when called. + """ + with pytest.raises(AttributeError): + Git.foo # type: ignore[attr-defined] + + +def test_get_use_shell_on_class_default() -> None: + """USE_SHELL can be read as a class attribute, defaulting to False and warning.""" + with pytest.deprecated_call() as ctx: + use_shell = Git.USE_SHELL + + (message,) = [str(entry.message) for entry in ctx] # Exactly one warning. + assert message.startswith(_USE_SHELL_DEPRECATED_FRAGMENT) + + assert_type(use_shell, bool) + + # This comes after the static assertion, just in case it would affect the inference. + assert not use_shell + + +# FIXME: More robustly check that each operation really issues exactly one deprecation +# warning, even if this requires relying more on reset_backing_attribute doing its job. +def test_use_shell_on_class(reset_backing_attribute) -> None: + """USE_SHELL can be written and re-read as a class attribute, always warning.""" + # We assert in a "safe" order, using reset_backing_attribute only as a backstop. + with pytest.deprecated_call() as ctx: + Git.USE_SHELL = True + set_value = Git.USE_SHELL + Git.USE_SHELL = False + reset_value = Git.USE_SHELL + + # The attribute should take on the values set to it. + assert set_value is True + assert reset_value is False + + messages = [str(entry.message) for entry in ctx] + set_message, check_message, reset_message, recheck_message = messages + + # Setting it to True should produce the special warning for that. + assert _USE_SHELL_DEPRECATED_FRAGMENT in set_message + assert set_message.startswith(_USE_SHELL_DANGEROUS_FRAGMENT) + + # All other operations should produce a usual warning. + assert check_message.startswith(_USE_SHELL_DEPRECATED_FRAGMENT) + assert reset_message.startswith(_USE_SHELL_DEPRECATED_FRAGMENT) + assert recheck_message.startswith(_USE_SHELL_DEPRECATED_FRAGMENT) + + +# FIXME: Test behavior on instances (where we can get but not set). + +# FIXME: Test behavior with multiprocessing (the attribute needs to pickle properly). + + +_EXPECTED_DIR_SUBSET = { + "cat_file_all", + "cat_file_header", + "GIT_PYTHON_TRACE", + "USE_SHELL", # The attribute we get deprecation warnings for. + "GIT_PYTHON_GIT_EXECUTABLE", + "refresh", + "is_cygwin", + "polish_url", + "check_unsafe_protocols", + "check_unsafe_options", + "AutoInterrupt", + "CatFileContentStream", + "__init__", + "__getattr__", + "set_persistent_git_options", + "working_dir", + "version_info", + "execute", + "environment", + "update_environment", + "custom_environment", + "transform_kwarg", + "transform_kwargs", + "__call__", + "_call_process", # Not currently considered public, but unlikely to change. + "get_object_header", + "get_object_data", + "stream_object_data", + "clear_cache", +} +"""Some stable attributes dir() should include on the Git class and its instances. + +This is intentionally incomplete, but includes substantial variety. Most importantly, it +includes both ``USE_SHELL`` and a wide sampling of other attributes. +""" + + +def test_class_dir() -> None: + """dir() on the Git class includes its statically known attributes. + + This tests that the mechanism that adds dynamic behavior to USE_SHELL accesses so + that all accesses issue warnings does not break dir() for the class, neither for + USE_SHELL nor for ordinary (non-deprecated) attributes. + """ + actual = set(dir(Git)) + assert _EXPECTED_DIR_SUBSET <= actual + + +def test_instance_dir() -> None: + """dir() on Git objects includes its statically known attributes. + + This is like test_class_dir, but for Git instance rather than the class itself. + """ + instance = Git() + actual = set(dir(instance)) + assert _EXPECTED_DIR_SUBSET <= actual From af723d5eb04dc919f76369c0b44a087638217352 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 24 Mar 2024 21:05:57 -0400 Subject: [PATCH 0951/1392] Make test_use_shell_on_class more robust --- test/deprecation/test_cmd_git.py | 54 ++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index fc8bab6ee..319bf7865 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -9,6 +9,7 @@ import contextlib import sys from typing import Generator +import warnings if sys.version_info >= (3, 11): from typing import assert_type @@ -26,9 +27,16 @@ """Beginning text of USE_SHELL deprecation warnings when USE_SHELL is set True.""" +@contextlib.contextmanager +def _suppress_deprecation_warning() -> Generator[None, None, None]: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + yield + + @pytest.fixture -def reset_backing_attribute() -> Generator[None, None, None]: - """Fixture to reset the private ``_USE_SHELL`` attribute. +def try_restore_use_shell_state() -> Generator[None, None, None]: + """Fixture to attempt to restore state associated with the ``USE_SHELL`` attribute. This is used to decrease the likelihood of state changes leaking out and affecting other tests. But the goal is not to assert that ``_USE_SHELL`` is used, nor anything @@ -38,18 +46,27 @@ def reset_backing_attribute() -> Generator[None, None, None]: restores attributes that it has previously been used to change, create, or remove. """ no_value = object() + try: - old_value = Git._USE_SHELL + old_backing_value = Git._USE_SHELL except AttributeError: - old_value = no_value + old_backing_value = no_value + try: + with _suppress_deprecation_warning(): + old_public_value = Git.USE_SHELL - yield + # This doesn't have its own try-finally because pytest catches exceptions raised + # during the yield. (The outer try-finally catches exceptions in this fixture.) + yield - if old_value is no_value: - with contextlib.suppress(AttributeError): - del Git._USE_SHELL - else: - Git._USE_SHELL = old_value + with _suppress_deprecation_warning(): + Git.USE_SHELL = old_public_value + finally: + if old_backing_value is no_value: + with contextlib.suppress(AttributeError): + del Git._USE_SHELL + else: + Git._USE_SHELL = old_backing_value def test_cannot_access_undefined_on_git_class() -> None: @@ -76,23 +93,26 @@ def test_get_use_shell_on_class_default() -> None: assert not use_shell -# FIXME: More robustly check that each operation really issues exactly one deprecation -# warning, even if this requires relying more on reset_backing_attribute doing its job. -def test_use_shell_on_class(reset_backing_attribute) -> None: +def test_use_shell_on_class(try_restore_use_shell_state) -> None: """USE_SHELL can be written and re-read as a class attribute, always warning.""" - # We assert in a "safe" order, using reset_backing_attribute only as a backstop. - with pytest.deprecated_call() as ctx: + with pytest.deprecated_call() as setting: Git.USE_SHELL = True + with pytest.deprecated_call() as checking: set_value = Git.USE_SHELL + with pytest.deprecated_call() as resetting: Git.USE_SHELL = False + with pytest.deprecated_call() as rechecking: reset_value = Git.USE_SHELL # The attribute should take on the values set to it. assert set_value is True assert reset_value is False - messages = [str(entry.message) for entry in ctx] - set_message, check_message, reset_message, recheck_message = messages + # Each access should warn exactly once. + (set_message,) = [str(entry.message) for entry in setting] + (check_message,) = [str(entry.message) for entry in checking] + (reset_message,) = [str(entry.message) for entry in resetting] + (recheck_message,) = [str(entry.message) for entry in rechecking] # Setting it to True should produce the special warning for that. assert _USE_SHELL_DEPRECATED_FRAGMENT in set_message From bf1388896ac2d052cf956123513f0c8b33f34dd6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Mar 2024 00:06:40 -0400 Subject: [PATCH 0952/1392] Write most remaining Git attribute/deprecation tests --- test/deprecation/test_cmd_git.py | 99 ++++++++++++++++++++++++++++---- 1 file changed, 88 insertions(+), 11 deletions(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index 319bf7865..ef6f5b5a6 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -17,6 +17,7 @@ from typing_extensions import assert_type import pytest +from pytest import WarningsRecorder from git.cmd import Git @@ -93,17 +94,34 @@ def test_get_use_shell_on_class_default() -> None: assert not use_shell -def test_use_shell_on_class(try_restore_use_shell_state) -> None: - """USE_SHELL can be written and re-read as a class attribute, always warning.""" - with pytest.deprecated_call() as setting: - Git.USE_SHELL = True - with pytest.deprecated_call() as checking: - set_value = Git.USE_SHELL - with pytest.deprecated_call() as resetting: - Git.USE_SHELL = False - with pytest.deprecated_call() as rechecking: - reset_value = Git.USE_SHELL +def test_get_use_shell_on_instance_default() -> None: + """USE_SHELL can be read as an instance attribute, defaulting to False and warning. + + This is the same as test_get_use_shell_on_class_default above, but for instances. + The test is repeated, instead of using parametrization, for clearer static analysis. + """ + instance = Git() + with pytest.deprecated_call() as ctx: + use_shell = instance.USE_SHELL + + (message,) = [str(entry.message) for entry in ctx] # Exactly one warning. + assert message.startswith(_USE_SHELL_DEPRECATED_FRAGMENT) + + assert_type(use_shell, bool) + + # This comes after the static assertion, just in case it would affect the inference. + assert not use_shell + + +def _assert_use_shell_full_results( + set_value: bool, + reset_value: bool, + setting: WarningsRecorder, + checking: WarningsRecorder, + resetting: WarningsRecorder, + rechecking: WarningsRecorder, +) -> None: # The attribute should take on the values set to it. assert set_value is True assert reset_value is False @@ -124,7 +142,66 @@ def test_use_shell_on_class(try_restore_use_shell_state) -> None: assert recheck_message.startswith(_USE_SHELL_DEPRECATED_FRAGMENT) -# FIXME: Test behavior on instances (where we can get but not set). +def test_use_shell_set_and_get_on_class(try_restore_use_shell_state: None) -> None: + """USE_SHELL can be set and re-read as a class attribute, always warning.""" + with pytest.deprecated_call() as setting: + Git.USE_SHELL = True + with pytest.deprecated_call() as checking: + set_value = Git.USE_SHELL + with pytest.deprecated_call() as resetting: + Git.USE_SHELL = False + with pytest.deprecated_call() as rechecking: + reset_value = Git.USE_SHELL + + _assert_use_shell_full_results( + set_value, + reset_value, + setting, + checking, + resetting, + rechecking, + ) + + +def test_use_shell_set_on_class_get_on_instance(try_restore_use_shell_state: None) -> None: + """USE_SHELL can be set on the class and read on an instance, always warning. + + This is like test_use_shell_set_and_get_on_class but it performs reads on an + instance. There is some redundancy here in assertions about warnings when the + attribute is set, but it is a separate test so that any bugs where a read on the + class (or an instance) is needed first before a read on an instance (or the class) + are detected. + """ + instance = Git() + + with pytest.deprecated_call() as setting: + Git.USE_SHELL = True + with pytest.deprecated_call() as checking: + set_value = instance.USE_SHELL + with pytest.deprecated_call() as resetting: + Git.USE_SHELL = False + with pytest.deprecated_call() as rechecking: + reset_value = instance.USE_SHELL + + _assert_use_shell_full_results( + set_value, + reset_value, + setting, + checking, + resetting, + rechecking, + ) + + +@pytest.mark.parametrize("value", [False, True]) +def test_use_shell_cannot_set_on_instance( + value: bool, + try_restore_use_shell_state: None, # In case of a bug where it does set USE_SHELL. +) -> None: + instance = Git() + with pytest.raises(AttributeError): + instance.USE_SHELL = value + # FIXME: Test behavior with multiprocessing (the attribute needs to pickle properly). From 602de0c93572972d29d33c4ecacfd9c6e192484a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Mar 2024 14:59:13 -0400 Subject: [PATCH 0953/1392] Begin multiprocessing misadventure There is no per-instance state involved in USE_SHELL, so pickling is far less directly relevant than usual to multiprocessing: the spawn and forkserver methods will not preserve a subsequently changed attribute value unless side effects of loading a module (or other unpickling of a function or its arguments that are submitted to run on a worker subprocess) causes it to run again; the fork method will. This will be (automatically) the same with any combination of metaclasses, properties, and custom descriptors as in the more straightforward case of a simple class attribute. Subtleties arise in the code that uses GitPython and multiprocessing, but should not arise unintentionally from the change in implementation of USE_SHELL done to add deprecation warnings, except possibly with respect to whether warnings will be repeated in worker processes, which is less important than whether the actual state is preserved. --- test/deprecation/test_cmd_git.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index ef6f5b5a6..72c1f7cd8 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -6,6 +6,7 @@ access are not inadvertently broken by mechanisms introduced to issue the warnings. """ +from concurrent.futures import ProcessPoolExecutor import contextlib import sys from typing import Generator @@ -36,7 +37,7 @@ def _suppress_deprecation_warning() -> Generator[None, None, None]: @pytest.fixture -def try_restore_use_shell_state() -> Generator[None, None, None]: +def restore_use_shell_state() -> Generator[None, None, None]: """Fixture to attempt to restore state associated with the ``USE_SHELL`` attribute. This is used to decrease the likelihood of state changes leaking out and affecting @@ -142,7 +143,7 @@ def _assert_use_shell_full_results( assert recheck_message.startswith(_USE_SHELL_DEPRECATED_FRAGMENT) -def test_use_shell_set_and_get_on_class(try_restore_use_shell_state: None) -> None: +def test_use_shell_set_and_get_on_class(restore_use_shell_state: None) -> None: """USE_SHELL can be set and re-read as a class attribute, always warning.""" with pytest.deprecated_call() as setting: Git.USE_SHELL = True @@ -163,7 +164,7 @@ def test_use_shell_set_and_get_on_class(try_restore_use_shell_state: None) -> No ) -def test_use_shell_set_on_class_get_on_instance(try_restore_use_shell_state: None) -> None: +def test_use_shell_set_on_class_get_on_instance(restore_use_shell_state: None) -> None: """USE_SHELL can be set on the class and read on an instance, always warning. This is like test_use_shell_set_and_get_on_class but it performs reads on an @@ -196,14 +197,31 @@ class (or an instance) is needed first before a read on an instance (or the clas @pytest.mark.parametrize("value", [False, True]) def test_use_shell_cannot_set_on_instance( value: bool, - try_restore_use_shell_state: None, # In case of a bug where it does set USE_SHELL. + restore_use_shell_state: None, # In case of a bug where it does set USE_SHELL. ) -> None: instance = Git() with pytest.raises(AttributeError): instance.USE_SHELL = value -# FIXME: Test behavior with multiprocessing (the attribute needs to pickle properly). +def _check_use_shell_in_worker(value: bool) -> None: + # USE_SHELL should have the value set in the parent before starting the worker. + assert Git.USE_SHELL is value + + # FIXME: Check that mutation still works and raises the warning. + + +@pytest.mark.filterwarnings("ignore::DeprecationWarning") +@pytest.mark.parametrize("value", [False, True]) +def test_use_shell_preserved_in_multiprocessing( + value: bool, + restore_use_shell_state: None, +) -> None: + """The USE_SHELL class attribute pickles accurately for multiprocessing.""" + Git.USE_SHELL = value + with ProcessPoolExecutor(max_workers=1) as executor: + # Calling result() marshals any exception back to this process and raises it. + executor.submit(_check_use_shell_in_worker, value).result() _EXPECTED_DIR_SUBSET = { From d4b50c94ff3694e3285a1ea8ed9b42c685726baf Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 25 Mar 2024 15:01:23 -0400 Subject: [PATCH 0954/1392] Somewhat clarify multiprocessing misadventure --- test/deprecation/test_cmd_git.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index 72c1f7cd8..04fb0747c 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -204,11 +204,8 @@ def test_use_shell_cannot_set_on_instance( instance.USE_SHELL = value -def _check_use_shell_in_worker(value: bool) -> None: - # USE_SHELL should have the value set in the parent before starting the worker. - assert Git.USE_SHELL is value - - # FIXME: Check that mutation still works and raises the warning. +def _get_value_in_current_process() -> bool: + return Git.USE_SHELL @pytest.mark.filterwarnings("ignore::DeprecationWarning") @@ -220,8 +217,8 @@ def test_use_shell_preserved_in_multiprocessing( """The USE_SHELL class attribute pickles accurately for multiprocessing.""" Git.USE_SHELL = value with ProcessPoolExecutor(max_workers=1) as executor: - # Calling result() marshals any exception back to this process and raises it. - executor.submit(_check_use_shell_in_worker, value).result() + marshaled_value = executor.submit(_get_value_in_current_process).result() + assert marshaled_value is value _EXPECTED_DIR_SUBSET = { From 02c2f0008082e710ddb73f1c4ff784b74eed4f8f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 26 Mar 2024 13:09:08 -0400 Subject: [PATCH 0955/1392] Discuss multiprocessing in test module docstring; remove bad test --- test/deprecation/test_cmd_git.py | 44 ++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index 04fb0747c..9dfe37220 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -1,12 +1,35 @@ """Tests for static and dynamic characteristics of Git class and instance attributes. -Currently this all relates to the deprecated :class:`Git.USE_SHELL` class attribute, +Currently this all relates to the deprecated :attr:`Git.USE_SHELL` class attribute, which can also be accessed through instances. Some tests directly verify its behavior, including deprecation warnings, while others verify that other aspects of attribute access are not inadvertently broken by mechanisms introduced to issue the warnings. + +A note on multiprocessing: Because USE_SHELL has no instance state, this module does not +include tests of pickling and multiprocessing. + +- Just as with a simple class attribute, when a class attribute with custom logic is + later set to a new value, it may have either its initial value or the new value when + accessed from a worker process, depending on the process start method. With "fork", + changes are preserved. With "spawn" or "forkserver", re-importing the modules causes + initial values to be set. Then the value in the parent at the time it dispatches the + task is only set in the children if the parent has the task set it, or if it is set as + a side effect of importing needed modules, or of unpickling objects passed to the + child (for example, if it is set in a top-level statement of the module that defines + the function submitted for the child worker process to call). + +- When an attribute gains new logic provided by a property or custom descriptor, and the + attribute involves instance-level state, incomplete or corrupted pickling can break + multiprocessing. (For example, if an instance attribute is reimplemented using a + descriptor that stores data in a global WeakKeyDictionary, pickled instances should be + tested to ensure they are still working correctly.) But nothing like that applies + here, because instance state is not involved. Although the situation is inherently + complex as described above, it is independent of the attribute implementation. + +- That USE_SHELL cannot be set on instances, and that when retrieved on instances it + always gives the same value as on the class, is covered in the tests here. """ -from concurrent.futures import ProcessPoolExecutor import contextlib import sys from typing import Generator @@ -204,23 +227,6 @@ def test_use_shell_cannot_set_on_instance( instance.USE_SHELL = value -def _get_value_in_current_process() -> bool: - return Git.USE_SHELL - - -@pytest.mark.filterwarnings("ignore::DeprecationWarning") -@pytest.mark.parametrize("value", [False, True]) -def test_use_shell_preserved_in_multiprocessing( - value: bool, - restore_use_shell_state: None, -) -> None: - """The USE_SHELL class attribute pickles accurately for multiprocessing.""" - Git.USE_SHELL = value - with ProcessPoolExecutor(max_workers=1) as executor: - marshaled_value = executor.submit(_get_value_in_current_process).result() - assert marshaled_value is value - - _EXPECTED_DIR_SUBSET = { "cat_file_all", "cat_file_header", From 46df79f75e6ee4db9852c398ec99a5788fd966b2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 26 Mar 2024 16:03:58 -0400 Subject: [PATCH 0956/1392] Discuss metaclass conflicts in test module docstring --- test/deprecation/test_cmd_git.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index 9dfe37220..a5d241cc4 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -5,8 +5,11 @@ including deprecation warnings, while others verify that other aspects of attribute access are not inadvertently broken by mechanisms introduced to issue the warnings. -A note on multiprocessing: Because USE_SHELL has no instance state, this module does not -include tests of pickling and multiprocessing. +A note on multiprocessing +========================= + +Because USE_SHELL has no instance state, this module does not include tests of pickling +and multiprocessing: - Just as with a simple class attribute, when a class attribute with custom logic is later set to a new value, it may have either its initial value or the new value when @@ -28,6 +31,26 @@ - That USE_SHELL cannot be set on instances, and that when retrieved on instances it always gives the same value as on the class, is covered in the tests here. + +A note on metaclass conflicts +============================= + +The most important DeprecationWarning is for the code ``Git.USE_SHELL = True``, which is +a security risk. But this warning may not be possible to implement without a custom +metaclass. This is because a descriptor in a class can customize all forms of attribute +access on its instances, but can only customize getting an attribute on the class. +Retrieving a descriptor from a class calls its ``__get__`` method (if defined), but +replacing or deleting it does not call its ``__set__`` or ``__delete__`` methods. + +Adding a metaclass is a potentially breaking change. This is because derived classes +that use an unrelated metaclass, whether directly or by inheriting from a class such as +abc.ABC that uses one, will raise TypeError when defined. These would have to be +modified to use a newly introduced metaclass that is a lower bound of both. Subclasses +remain unbroken in the far more typical case that they use no custom metaclass. + +The tests in this module do not establish whether the danger of setting Git.USE_SHELL to +True is high enough, and applications of deriving from Git and using an unrelated custom +metaclass marginal enough, to justify introducing a metaclass to issue the warnings. """ import contextlib From 40ed842cf35b0c280dc408d77c764988a2d2746a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 26 Mar 2024 18:30:34 -0400 Subject: [PATCH 0957/1392] Revise test module docstring for clarity --- test/deprecation/test_cmd_git.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index a5d241cc4..2ebcb3ad7 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -11,19 +11,19 @@ Because USE_SHELL has no instance state, this module does not include tests of pickling and multiprocessing: -- Just as with a simple class attribute, when a class attribute with custom logic is - later set to a new value, it may have either its initial value or the new value when - accessed from a worker process, depending on the process start method. With "fork", - changes are preserved. With "spawn" or "forkserver", re-importing the modules causes - initial values to be set. Then the value in the parent at the time it dispatches the - task is only set in the children if the parent has the task set it, or if it is set as - a side effect of importing needed modules, or of unpickling objects passed to the - child (for example, if it is set in a top-level statement of the module that defines - the function submitted for the child worker process to call). +- Just as with a simple class attribute, when a class attribute with custom logic is set + to another value, even before a worker process is created that uses the class, the + worker process may see either the initial or new value, depending on the process start + method. With "fork", changes are preserved. With "spawn" or "forkserver", re-importing + the modules causes initial values to be set. Then the value in the parent at the time + it dispatches the task is only set in the children if the parent has the task set it, + or if it is set as a side effect of importing needed modules, or of unpickling objects + passed to the child (for example, if it is set in a top-level statement of the module + that defines the function submitted for the child worker process to call). - When an attribute gains new logic provided by a property or custom descriptor, and the attribute involves instance-level state, incomplete or corrupted pickling can break - multiprocessing. (For example, if an instance attribute is reimplemented using a + multiprocessing. (For example, when an instance attribute is reimplemented using a descriptor that stores data in a global WeakKeyDictionary, pickled instances should be tested to ensure they are still working correctly.) But nothing like that applies here, because instance state is not involved. Although the situation is inherently @@ -35,8 +35,8 @@ A note on metaclass conflicts ============================= -The most important DeprecationWarning is for the code ``Git.USE_SHELL = True``, which is -a security risk. But this warning may not be possible to implement without a custom +The most important DeprecationWarning is for code like ``Git.USE_SHELL = True``, which +is a security risk. But this warning may not be possible to implement without a custom metaclass. This is because a descriptor in a class can customize all forms of attribute access on its instances, but can only customize getting an attribute on the class. Retrieving a descriptor from a class calls its ``__get__`` method (if defined), but From 6a35261ab6a58bfd8fbb2b008aacfb2decf27053 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 03:28:38 -0400 Subject: [PATCH 0958/1392] Test that USE_SHELL is unittest.mock.patch patchable --- test/deprecation/test_cmd_git.py | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index 2ebcb3ad7..5d710b0d4 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -56,6 +56,7 @@ import contextlib import sys from typing import Generator +import unittest.mock import warnings if sys.version_info >= (3, 11): @@ -250,6 +251,41 @@ def test_use_shell_cannot_set_on_instance( instance.USE_SHELL = value +@pytest.mark.filterwarnings("ignore::DeprecationWarning") +@pytest.mark.parametrize("original_value", [False, True]) +def test_use_shell_is_mock_patchable_on_class_as_object_attribute( + original_value: bool, + restore_use_shell_state: None, +) -> None: + """Asymmetric patching looking up USE_SHELL in ``__dict__`` doesn't corrupt state. + + Code using GitPython may temporarily set Git.USE_SHELL to a different value. Ideally + it does not use unittest.mock.patch to do so, because that makes subtle assumptions + about the relationship between attributes and dictionaries. If the attribute can be + retrieved from the ``__dict__`` rather than directly, that value is assumed the + correct one to restore, even by a normal setattr. + + The effect is that some ways of simulating a class attribute with added behavior can + cause a descriptor, such as a property, to be set to its own backing attribute + during unpatching; then subsequent reads raise RecursionError. This happens if both + (a) setting it on the class is customized in a metaclass and (b) getting it on + instances is customized with a descriptor (such as a property) in the class itself. + + Although ideally code outside GitPython would not rely on being able to patch + Git.USE_SHELL with unittest.mock.patch, the technique is widespread. Thus, USE_SHELL + should be implemented in some way compatible with it. This test checks for that. + """ + Git.USE_SHELL = original_value + if Git.USE_SHELL is not original_value: + raise RuntimeError(f"Can't set up the test") + new_value = not original_value + + with unittest.mock.patch.object(Git, "USE_SHELL", new_value): + assert Git.USE_SHELL is new_value + + assert Git.USE_SHELL is original_value + + _EXPECTED_DIR_SUBSET = { "cat_file_all", "cat_file_header", From e725c8254cccd88d02b38b414bac875212a7074b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 11:45:25 -0400 Subject: [PATCH 0959/1392] Make the restore_use_shell_state fixture more robust --- test/deprecation/test_cmd_git.py | 52 ++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index 5d710b0d4..4183018e6 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -54,6 +54,7 @@ """ import contextlib +import logging import sys from typing import Generator import unittest.mock @@ -75,6 +76,8 @@ _USE_SHELL_DANGEROUS_FRAGMENT = "Setting Git.USE_SHELL to True is unsafe and insecure" """Beginning text of USE_SHELL deprecation warnings when USE_SHELL is set True.""" +_logger = logging.getLogger(__name__) + @contextlib.contextmanager def _suppress_deprecation_warning() -> Generator[None, None, None]: @@ -85,22 +88,44 @@ def _suppress_deprecation_warning() -> Generator[None, None, None]: @pytest.fixture def restore_use_shell_state() -> Generator[None, None, None]: - """Fixture to attempt to restore state associated with the ``USE_SHELL`` attribute. + """Fixture to attempt to restore state associated with the USE_SHELL attribute. This is used to decrease the likelihood of state changes leaking out and affecting - other tests. But the goal is not to assert that ``_USE_SHELL`` is used, nor anything - about how or when it is used, which is an implementation detail subject to change. + other tests. But the goal is not to assert implementation details of USE_SHELL. + + This covers two of the common implementation strategies, for convenience in testing + both. USE_SHELL could be implemented in the metaclass: - This is possible but inelegant to do with pytest's monkeypatch fixture, which only - restores attributes that it has previously been used to change, create, or remove. + * With a separate _USE_SHELL backing attribute. If using a property or other + descriptor, this is the natural way to do it, but custom __getattribute__ and + __setattr__ logic, if it does more than adding warnings, may also use that. + * Like a simple attribute, using USE_SHELL itself, stored as usual in the class + dictionary, with custom __getattribute__/__setattr__ logic only to warn. + + This tries to save private state, tries to save the public attribute value, yields + to the test case, tries to restore the public attribute value, then tries to restore + private state. The idea is that if the getting or setting logic is wrong in the code + under test, the state will still most likely be reset successfully. """ no_value = object() + # Try to save the original private state. try: - old_backing_value = Git._USE_SHELL + old_private_value = Git._USE_SHELL except AttributeError: - old_backing_value = no_value + separate_backing_attribute = False + try: + old_private_value = type.__getattribute__(Git, "USE_SHELL") + except AttributeError: + old_private_value = no_value + _logger.error("Cannot retrieve old private _USE_SHELL or USE_SHELL value") + else: + separate_backing_attribute = True + try: + # Try to save the original public value. Rather than attempt to restore a state + # where the attribute is not set, if we cannot do this we allow AttributeError + # to propagate out of the fixture, erroring the test case before its code runs. with _suppress_deprecation_warning(): old_public_value = Git.USE_SHELL @@ -108,14 +133,15 @@ def restore_use_shell_state() -> Generator[None, None, None]: # during the yield. (The outer try-finally catches exceptions in this fixture.) yield + # Try to restore the original public value. with _suppress_deprecation_warning(): Git.USE_SHELL = old_public_value finally: - if old_backing_value is no_value: - with contextlib.suppress(AttributeError): - del Git._USE_SHELL - else: - Git._USE_SHELL = old_backing_value + # Try to restore the original private state. + if separate_backing_attribute: + Git._USE_SHELL = old_private_value + elif old_private_value is not no_value: + type.__setattr__(Git, "USE_SHELL", old_private_value) def test_cannot_access_undefined_on_git_class() -> None: @@ -277,7 +303,7 @@ def test_use_shell_is_mock_patchable_on_class_as_object_attribute( """ Git.USE_SHELL = original_value if Git.USE_SHELL is not original_value: - raise RuntimeError(f"Can't set up the test") + raise RuntimeError("Can't set up the test") new_value = not original_value with unittest.mock.patch.object(Git, "USE_SHELL", new_value): From 436bcaa85712f73640ac719679d6d1366c376483 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 12:05:52 -0400 Subject: [PATCH 0960/1392] Add `type: ignore` in test that we can't set USE_SHELL on instances As with other `type: ignore` comments in the deprecation tests, mypy will detect if it is superfluous (provided warn_unused_ignores is set to true), thereby enforcing that such code is a static type error. --- test/deprecation/test_cmd_git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index 4183018e6..4925fada8 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -274,7 +274,7 @@ def test_use_shell_cannot_set_on_instance( ) -> None: instance = Git() with pytest.raises(AttributeError): - instance.USE_SHELL = value + instance.USE_SHELL = value # type: ignore[misc] # Name not in __slots__. @pytest.mark.filterwarnings("ignore::DeprecationWarning") From febda6f6e9aa56f4b525020153ed88459906641f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 12:09:15 -0400 Subject: [PATCH 0961/1392] Clarify unittest.mock.patch patchability test docstring --- test/deprecation/test_cmd_git.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index 4925fada8..b792fddb8 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -292,10 +292,11 @@ def test_use_shell_is_mock_patchable_on_class_as_object_attribute( correct one to restore, even by a normal setattr. The effect is that some ways of simulating a class attribute with added behavior can - cause a descriptor, such as a property, to be set to its own backing attribute - during unpatching; then subsequent reads raise RecursionError. This happens if both - (a) setting it on the class is customized in a metaclass and (b) getting it on - instances is customized with a descriptor (such as a property) in the class itself. + cause a descriptor, such as a property, to be set as the value of its own backing + attribute during unpatching; then subsequent reads raise RecursionError. This + happens if both (a) setting it on the class is customized in a metaclass and (b) + getting it on instances is customized with a descriptor (such as a property) in the + class itself. Although ideally code outside GitPython would not rely on being able to patch Git.USE_SHELL with unittest.mock.patch, the technique is widespread. Thus, USE_SHELL From 40371087ac83497f307d2239f36646a321028829 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 13:31:22 -0400 Subject: [PATCH 0962/1392] Test that Git.execute's own read of USE_SHELL does not warn + Use simplefilter where we can (separate from this test). --- test/deprecation/test_cmd_git.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index b792fddb8..f17756fb6 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -82,7 +82,7 @@ @contextlib.contextmanager def _suppress_deprecation_warning() -> Generator[None, None, None]: with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=DeprecationWarning) + warnings.simplefilter("ignore", DeprecationWarning) yield @@ -313,6 +313,24 @@ class itself. assert Git.USE_SHELL is original_value +def test_execute_without_shell_arg_does_not_warn() -> None: + """No deprecation warning is issued from operations implemented using Git.execute(). + + When no ``shell`` argument is passed to Git.execute, which is when the value of + USE_SHELL is to be used, the way Git.execute itself accesses USE_SHELL does not + issue a deprecation warning. + """ + with warnings.catch_warnings(): + for category in DeprecationWarning, PendingDeprecationWarning: + warnings.filterwarnings( + action="error", + category=category, + module=r"git(?:\.|$)", + ) + + Git().version() + + _EXPECTED_DIR_SUBSET = { "cat_file_all", "cat_file_header", From 0e311bf5da4332acd314aa6927138b36bbd9e527 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 14:42:26 -0400 Subject: [PATCH 0963/1392] Suppress type errors in restore_use_shell_state _USE_SHELL branches These conditional branches are kept so alternative implementations can be examined, including if they need to be investigated to satisfy some future requirement. But to be unittest.mock.patch patchable, the approaches that would have a _USE_SHELL backing attribute would be difficult to implement in a straightforward way, which seems not to be needed or justified at this time. Since that is not anticipated (except as an intermediate step in development), these suppressions make sense, and they will also be reported by mypy if the implementation changes to benefit from them (so long as it is configured with warn_unused_ignores set to true). --- test/deprecation/test_cmd_git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index f17756fb6..b6af5c770 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -111,7 +111,7 @@ def restore_use_shell_state() -> Generator[None, None, None]: # Try to save the original private state. try: - old_private_value = Git._USE_SHELL + old_private_value = Git._USE_SHELL # type: ignore[attr-defined] except AttributeError: separate_backing_attribute = False try: @@ -139,7 +139,7 @@ def restore_use_shell_state() -> Generator[None, None, None]: finally: # Try to restore the original private state. if separate_backing_attribute: - Git._USE_SHELL = old_private_value + Git._USE_SHELL = old_private_value # type: ignore[attr-defined] elif old_private_value is not no_value: type.__setattr__(Git, "USE_SHELL", old_private_value) From df4c5c03d14a53448a232a78eeab94017ac7e7ba Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Mar 2024 15:33:47 -0400 Subject: [PATCH 0964/1392] Fix wrong/unclear grammar in test_instance_dir docstring --- test/deprecation/test_cmd_git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index b6af5c770..d312b8778 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -383,7 +383,7 @@ def test_class_dir() -> None: def test_instance_dir() -> None: """dir() on Git objects includes its statically known attributes. - This is like test_class_dir, but for Git instance rather than the class itself. + This is like test_class_dir, but for Git instances rather than the class itself. """ instance = Git() actual = set(dir(instance)) From d38e721c1f35695ecdaf6e88b721881fb7b3ed6f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Tue, 26 Mar 2024 21:55:04 -0400 Subject: [PATCH 0965/1392] Issue warnings whenever Git.USE_SHELL is accessed With a special message when it is assigned a True value, which is the dangerous use that underlies its deprecation. The warnings are all DeprecationWarning. --- git/cmd.py | 78 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 17 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 03e5d7ffc..0b7536d91 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -19,6 +19,7 @@ import sys from textwrap import dedent import threading +import warnings from git.compat import defenc, force_bytes, safe_decode from git.exc import ( @@ -54,6 +55,7 @@ TYPE_CHECKING, TextIO, Tuple, + Type, Union, cast, overload, @@ -307,8 +309,45 @@ def dict_to_slots_and__excluded_are_none(self: object, d: Mapping[str, Any], exc ## -- End Utilities -- @} +_USE_SHELL_DEFAULT_MESSAGE = ( + "Git.USE_SHELL is deprecated, because only its default value of False is safe. " + "It will be removed in a future release." +) + +_USE_SHELL_DANGER_MESSAGE = ( + "Setting Git.USE_SHELL to True is unsafe and insecure, and should be avoided, " + "because the effect of shell metacharacters and shell expansions cannot usually be " + "accounted for. In addition, Git.USE_SHELL is deprecated and will be removed in a " + "future release." +) + + +def _warn_use_shell(extra_danger: bool) -> None: + warnings.warn( + _USE_SHELL_DANGER_MESSAGE if extra_danger else _USE_SHELL_DEFAULT_MESSAGE, + DeprecationWarning, + stacklevel=3, + ) + + +class _GitMeta(type): + """Metaclass for :class:`Git`. + + This helps issue :class:`DeprecationWarning` if :attr:`Git.USE_SHELL` is used. + """ -class Git: + @property + def USE_SHELL(cls: Type[Git]) -> bool: + _warn_use_shell(False) + return cls._USE_SHELL + + @USE_SHELL.setter + def USE_SHELL(cls: Type[Git], value: bool) -> None: + _warn_use_shell(value) + cls._USE_SHELL = value + + +class Git(metaclass=_GitMeta): """The Git class manages communication with the Git binary. It provides a convenient interface to calling the Git binary, such as in:: @@ -358,25 +397,30 @@ def __setstate__(self, d: Dict[str, Any]) -> None: GIT_PYTHON_TRACE = os.environ.get("GIT_PYTHON_TRACE", False) """Enables debugging of GitPython's git commands.""" - USE_SHELL = False - """Deprecated. If set to ``True``, a shell will be used when executing git commands. + _USE_SHELL: bool = False - Prior to GitPython 2.0.8, this had a narrow purpose in suppressing console windows - in graphical Windows applications. In 2.0.8 and higher, it provides no benefit, as - GitPython solves that problem more robustly and safely by using the - ``CREATE_NO_WINDOW`` process creation flag on Windows. + @property + def USE_SHELL(self) -> bool: + """Deprecated. If set to ``True``, a shell will be used to execute git commands. - Code that uses ``USE_SHELL = True`` or that passes ``shell=True`` to any GitPython - functions should be updated to use the default value of ``False`` instead. ``True`` - is unsafe unless the effect of shell expansions is fully considered and accounted - for, which is not possible under most circumstances. + Prior to GitPython 2.0.8, this had a narrow purpose in suppressing console + windows in graphical Windows applications. In 2.0.8 and higher, it provides no + benefit, as GitPython solves that problem more robustly and safely by using the + ``CREATE_NO_WINDOW`` process creation flag on Windows. - See: + Code that uses ``USE_SHELL = True`` or that passes ``shell=True`` to any + GitPython functions should be updated to use the default value of ``False`` + instead. ``True`` is unsafe unless the effect of shell expansions is fully + considered and accounted for, which is not possible under most circumstances. - - :meth:`Git.execute` (on the ``shell`` parameter). - - https://github.com/gitpython-developers/GitPython/commit/0d9390866f9ce42870d3116094cd49e0019a970a - - https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags - """ + See: + + - :meth:`Git.execute` (on the ``shell`` parameter). + - https://github.com/gitpython-developers/GitPython/commit/0d9390866f9ce42870d3116094cd49e0019a970a + - https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags + """ + _warn_use_shell(False) + return self._USE_SHELL _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" _refresh_env_var = "GIT_PYTHON_REFRESH" @@ -1138,7 +1182,7 @@ def execute( stdout_sink = PIPE if with_stdout else getattr(subprocess, "DEVNULL", None) or open(os.devnull, "wb") if shell is None: - shell = self.USE_SHELL + shell = self._USE_SHELL _logger.debug( "Popen(%s, cwd=%s, stdin=%s, shell=%s, universal_newlines=%s)", redacted_command, From 05de5c0d3bc81f4d56283399417db2f6927fe233 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 03:29:41 -0400 Subject: [PATCH 0966/1392] Implement instance USE_SHELL lookup in __getattr__ This is with the intention of making it so that Git.USE_SHELL is unittest.mock.patch patchable. However, while this moves in the right direction, it can't be done this way, as the attribute is found to be absent on the class, so when unittest.mock.patch unpatches, it tries to delete the attribute it has set, which fails due to the metaclass's USE_SHELL instance property having no deleter. --- git/cmd.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 0b7536d91..d01c15b04 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -399,28 +399,25 @@ def __setstate__(self, d: Dict[str, Any]) -> None: _USE_SHELL: bool = False - @property - def USE_SHELL(self) -> bool: - """Deprecated. If set to ``True``, a shell will be used to execute git commands. + USE_SHELL: bool + """Deprecated. If set to ``True``, a shell will be used to execute git commands. - Prior to GitPython 2.0.8, this had a narrow purpose in suppressing console - windows in graphical Windows applications. In 2.0.8 and higher, it provides no - benefit, as GitPython solves that problem more robustly and safely by using the - ``CREATE_NO_WINDOW`` process creation flag on Windows. + Prior to GitPython 2.0.8, this had a narrow purpose in suppressing console windows + in graphical Windows applications. In 2.0.8 and higher, it provides no benefit, as + GitPython solves that problem more robustly and safely by using the + ``CREATE_NO_WINDOW`` process creation flag on Windows. - Code that uses ``USE_SHELL = True`` or that passes ``shell=True`` to any - GitPython functions should be updated to use the default value of ``False`` - instead. ``True`` is unsafe unless the effect of shell expansions is fully - considered and accounted for, which is not possible under most circumstances. + Code that uses ``USE_SHELL = True`` or that passes ``shell=True`` to any GitPython + functions should be updated to use the default value of ``False`` instead. ``True`` + is unsafe unless the effect of shell expansions is fully considered and accounted + for, which is not possible under most circumstances. - See: + See: - - :meth:`Git.execute` (on the ``shell`` parameter). - - https://github.com/gitpython-developers/GitPython/commit/0d9390866f9ce42870d3116094cd49e0019a970a - - https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags - """ - _warn_use_shell(False) - return self._USE_SHELL + - :meth:`Git.execute` (on the ``shell`` parameter). + - https://github.com/gitpython-developers/GitPython/commit/0d9390866f9ce42870d3116094cd49e0019a970a + - https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags + """ _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" _refresh_env_var = "GIT_PYTHON_REFRESH" @@ -921,6 +918,11 @@ def __getattr__(self, name: str) -> Any: """ if name.startswith("_"): return super().__getattribute__(name) + + if name == "USE_SHELL": + _warn_use_shell(False) + return self._USE_SHELL + return lambda *args, **kwargs: self._call_process(name, *args, **kwargs) def set_persistent_git_options(self, **kwargs: Any) -> None: From 04eb09c728fbdc4501fbbba1d6f58a0bb7050470 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 03:40:14 -0400 Subject: [PATCH 0967/1392] Have USE_SHELL warn but work like normal via super() This reimplements Git.USE_SHELL with no properties or descriptors. The metaclass is still needed, but instad of defining properties it defines __getattribute__ and __setattr__, which check for USE_SHELL and warn, then invoke the default attribute access via super(). Likewise, in the Git class itself, a __getatttribute__ override is introduced (not to be confused with __getattr__, which was already present and handles attribute access when an attribute is otherwise absent, unlike __getattribute__ which is always used). This checks for reading USE_SHELL on an instance and warns, then invokes the default attribute access via super(). Advantages: - Git.USE_SHELL is again unittest.mock.patch patchable. - AttributeError messages are automatically as before. - It effectively is a simple attribute, yet with warning, so other unanticipated ways of accessing it may be less likely to break. - The code is simpler, cleaner, and clearer. There is some overhead, but it is small, especially compared to a subprocess invocation as is done for performing most git operations. However, this does introduce disadvantages that must be addressed: - Although attribute access on Git instances was already highly dynamic, as "methods" are synthesized for git subcommands, this was and is not the case for the Git class itself, whose attributes remain exactly those that can be inferred without considering the existence of __getattribute__ and __setattr__ on the metaclass. So static type checkers need to be prevented from accounting for those metaclass methods in a way that causes them to infer that arbitrary class attribute access is allowed. - The occurrence of Git.USE_SHELL in the Git.execute method (where the USE_SHELL attribute is actually examined) should be changed so it does not itself issue DeprecationWarning (it is not enough that by default a DeprecationWarning from there isn't displayed). --- git/cmd.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index d01c15b04..1c2aebd3c 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -55,7 +55,6 @@ TYPE_CHECKING, TextIO, Tuple, - Type, Union, cast, overload, @@ -336,15 +335,15 @@ class _GitMeta(type): This helps issue :class:`DeprecationWarning` if :attr:`Git.USE_SHELL` is used. """ - @property - def USE_SHELL(cls: Type[Git]) -> bool: - _warn_use_shell(False) - return cls._USE_SHELL + def __getattribute__(cls, name: str) -> Any: + if name == "USE_SHELL": + _warn_use_shell(False) + return super().__getattribute__(name) - @USE_SHELL.setter - def USE_SHELL(cls: Type[Git], value: bool) -> None: - _warn_use_shell(value) - cls._USE_SHELL = value + def __setattr__(cls, name: str, value: Any) -> Any: + if name == "USE_SHELL": + _warn_use_shell(value) + super().__setattr__(name, value) class Git(metaclass=_GitMeta): @@ -397,9 +396,7 @@ def __setstate__(self, d: Dict[str, Any]) -> None: GIT_PYTHON_TRACE = os.environ.get("GIT_PYTHON_TRACE", False) """Enables debugging of GitPython's git commands.""" - _USE_SHELL: bool = False - - USE_SHELL: bool + USE_SHELL: bool = False """Deprecated. If set to ``True``, a shell will be used to execute git commands. Prior to GitPython 2.0.8, this had a narrow purpose in suppressing console windows @@ -909,6 +906,11 @@ def __init__(self, working_dir: Union[None, PathLike] = None) -> None: self.cat_file_header: Union[None, TBD] = None self.cat_file_all: Union[None, TBD] = None + def __getattribute__(self, name: str) -> Any: + if name == "USE_SHELL": + _warn_use_shell(False) + return super().__getattribute__(name) + def __getattr__(self, name: str) -> Any: """A convenience method as it allows to call the command as if it was an object. @@ -918,11 +920,6 @@ def __getattr__(self, name: str) -> Any: """ if name.startswith("_"): return super().__getattribute__(name) - - if name == "USE_SHELL": - _warn_use_shell(False) - return self._USE_SHELL - return lambda *args, **kwargs: self._call_process(name, *args, **kwargs) def set_persistent_git_options(self, **kwargs: Any) -> None: @@ -1184,7 +1181,7 @@ def execute( stdout_sink = PIPE if with_stdout else getattr(subprocess, "DEVNULL", None) or open(os.devnull, "wb") if shell is None: - shell = self._USE_SHELL + shell = self.USE_SHELL _logger.debug( "Popen(%s, cwd=%s, stdin=%s, shell=%s, universal_newlines=%s)", redacted_command, From c6f518b40e42d394dfbcf338aacf101cad58b700 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 14:14:52 -0400 Subject: [PATCH 0968/1392] Keep mypy from thinking Git has arbitrary class attributes The existence of __getattr__ or __getattribute__ on a class causes static type checkers like mypy to stop inferring that reads of unrecognized instance attributes are static type errors. When the class is a metaclass, this causes static type checkers to stop inferring that reads of unrecognized class attributes, on classes that use (i.e., that have as their type) the metaclass, are static type errors. The Git class itself defines __getattr__ and __getattribute__, but Git objects' instance attributes really are dynamically synthesized (in __getattr__). However, class attributes of Git are not dynamic, even though _GitMeta defines __getattribute__. Therefore, something special is needed so mypy infers nothing about Git class attributes from the existence of _GitMeta.__getattribute__. This takes the same approach as taken to the analogous problem with __getattr__ at module level, defining __getattribute__ with a different name first and then assigning that to __getattribute__ under an `if not TYPE_CHECKING:`. (Allowing static type checkers to see the original definition allows them to find some kinds of type errors in the body of the method, which is why the method isn't just defined in the `if not TYPE_CHECKING`.) Although it may not currently be necessary to hide __setattr__ too, the same is done with it in _GitMeta as well. The reason is that the intent may otherwise be subtly amgiguous to human readers and maybe future tools: when __setattr__ exists, the effect of setting a class attribute that did not previously exist is in principle unclear, and might not make the attribute readble. (I am unsure if this is the right choice; either approach seems reasonable.) --- git/cmd.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 1c2aebd3c..ce19e733c 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -335,16 +335,23 @@ class _GitMeta(type): This helps issue :class:`DeprecationWarning` if :attr:`Git.USE_SHELL` is used. """ - def __getattribute__(cls, name: str) -> Any: + def __getattribute(cls, name: str) -> Any: if name == "USE_SHELL": _warn_use_shell(False) return super().__getattribute__(name) - def __setattr__(cls, name: str, value: Any) -> Any: + def __setattr(cls, name: str, value: Any) -> Any: if name == "USE_SHELL": _warn_use_shell(value) super().__setattr__(name, value) + if not TYPE_CHECKING: + # To preserve static checking for undefined/misspelled attributes while letting + # the methods' bodies be type-checked, these are defined as non-special methods, + # then bound to special names out of view of static type checkers. + __getattribute__ = __getattribute + __setattr__ = __setattr + class Git(metaclass=_GitMeta): """The Git class manages communication with the Git binary. From c5d5b16bfb4829dd510e3ff258b3daca4cd3b57d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 14:20:54 -0400 Subject: [PATCH 0969/1392] Clarify that the private name mangling is intentional --- git/cmd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index ce19e733c..de5c6929b 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -348,7 +348,8 @@ def __setattr(cls, name: str, value: Any) -> Any: if not TYPE_CHECKING: # To preserve static checking for undefined/misspelled attributes while letting # the methods' bodies be type-checked, these are defined as non-special methods, - # then bound to special names out of view of static type checkers. + # then bound to special names out of view of static type checkers. (The original + # names invoke name mangling (leading "__") to avoid confusion in other scopes.) __getattribute__ = __getattribute __setattr__ = __setattr From 84bf2ca034721cd54e792f57585083a1dbffc6ea Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 15:25:20 -0400 Subject: [PATCH 0970/1392] Read USE_SHELL in Git.execute without DeprecationWarning This changes how Git.execute itself accesses the USE_SHELL attribute, so that its own read of it does not issue a warning. --- git/cmd.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index de5c6929b..eed82d7dd 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1189,7 +1189,12 @@ def execute( stdout_sink = PIPE if with_stdout else getattr(subprocess, "DEVNULL", None) or open(os.devnull, "wb") if shell is None: - shell = self.USE_SHELL + # Get the value of USE_SHELL with no deprecation warning. Do this without + # warnings.catch_warnings, to avoid a race condition with application code + # configuring warnings. The value could be looked up in type(self).__dict__ + # or Git.__dict__, but those can break under some circumstances. This works + # the same as self.USE_SHELL in more situations; see Git.__getattribute__. + shell = super().__getattribute__("USE_SHELL") _logger.debug( "Popen(%s, cwd=%s, stdin=%s, shell=%s, universal_newlines=%s)", redacted_command, From 5bef7ed7369e1d6f93161607e01dc02bcd1720ab Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 28 Mar 2024 23:35:03 -0400 Subject: [PATCH 0971/1392] Add GitPython project top comments to new test modules --- test/deprecation/test_cmd_git.py | 3 +++ test/deprecation/test_compat.py | 3 +++ test/deprecation/test_toplevel.py | 3 +++ test/deprecation/test_types.py | 3 +++ 4 files changed, 12 insertions(+) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index d312b8778..b15f219d8 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Tests for static and dynamic characteristics of Git class and instance attributes. Currently this all relates to the deprecated :attr:`Git.USE_SHELL` class attribute, diff --git a/test/deprecation/test_compat.py b/test/deprecation/test_compat.py index 22f733083..2d7805e61 100644 --- a/test/deprecation/test_compat.py +++ b/test/deprecation/test_compat.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Tests for dynamic and static characteristics of git.compat module attributes. These tests verify that the is_ attributes are available, and are even listed diff --git a/test/deprecation/test_toplevel.py b/test/deprecation/test_toplevel.py index 54dc8e358..740408193 100644 --- a/test/deprecation/test_toplevel.py +++ b/test/deprecation/test_toplevel.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Tests for dynamic and static characteristics of top-level git module attributes. Provided mypy has ``warn_unused_ignores = true`` set, running mypy on these test cases diff --git a/test/deprecation/test_types.py b/test/deprecation/test_types.py index cd53fa210..f97375a85 100644 --- a/test/deprecation/test_types.py +++ b/test/deprecation/test_types.py @@ -1,3 +1,6 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + """Tests for dynamic and static characteristics of git.types module attributes.""" import sys From 3da47c26db44f287fa2b10e95995689de1f578bc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 29 Mar 2024 11:46:38 -0400 Subject: [PATCH 0972/1392] Hide `del util` from type checkers Even though current type checkers don't seem to need it. As noted in dffa930, it appears neither mypy nor pyright currently needs `del util` to be guarded by `if not TYPE_CHECKING:` -- they currently treat util as bound even without it, even with `del util` present. It is not obvious that this is the best type checker behavior or that type checkers will continue to behave this way. (In addition, it may help humans for all statements whose effects are intended not to be considered for purposes of static typing to be guarded by `if not TYPE_CHECKING:`.) So this guards the deletion of util the same as the binding of __getattr__. This also moves, clarifies, and expands the commented explanations of what is going on. --- git/__init__.py | 54 +++++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/git/__init__.py b/git/__init__.py index fd1b87707..1b2360e3a 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -169,6 +169,8 @@ IndexEntry, IndexFile, StageType, + # NOTE: This tells type checkers what util resolves to. We delete it, and it is + # really resolved by __getattr__, which warns. See below on what to use instead. util, ) from git.util import ( # @NoMove @@ -183,27 +185,6 @@ raise ImportError("%s: %s" % (_exc.__class__.__name__, _exc)) from _exc -# NOTE: The expression `git.util` evaluates to git.index.util and `from git import util` -# imports git.index.util, NOT git.util. It may not be feasible to change this until the -# next major version, to avoid breaking code inadvertently relying on it. -# -# - If git.index.util *is* what you want, use or import from that, to avoid confusion. -# -# - To use the "real" git.util module, write `from git.util import ...`, or if necessary -# access it as `sys.modules["git.util"]`. -# -# (This differs from other indirect-submodule imports that are unambiguously non-public -# and subject to immediate removal. Here, the public git.util module, though different, -# makes less discoverable that the expression `git.util` refers to a non-public -# attribute of the git module.) -# -# This had come about by a wildcard import. Now that all intended imports are explicit, -# the intuitive but potentially incompatible binding occurs due to the usual rules for -# Python submodule bindings. So for now we delete that and let __getattr__ handle it. -# -del util - - def _warned_import(message: str, fullname: str) -> "ModuleType": import importlib @@ -242,7 +223,36 @@ def _getattr(name: str) -> Any: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") -if not TYPE_CHECKING: # Preserve static checking for undefined/misspelled attributes. +if not TYPE_CHECKING: + # NOTE: The expression `git.util` gives git.index.util and `from git import util` + # imports git.index.util, NOT git.util. It may not be feasible to change this until + # the next major version, to avoid breaking code inadvertently relying on it. + # + # - If git.index.util *is* what you want, use (or import from) that, to avoid + # confusion. + # + # - To use the "real" git.util module, write `from git.util import ...`, or if + # necessary access it as `sys.modules["git.util"]`. + # + # Note also that `import git.util` technically imports the "real" git.util... but + # the *expression* `git.util` after doing so is still git.index.util! + # + # (This situation differs from that of other indirect-submodule imports that are + # unambiguously non-public and subject to immediate removal. Here, the public + # git.util module, though different, makes less discoverable that the expression + # `git.util` refers to a non-public attribute of the git module.) + # + # This had originally come about by a wildcard import. Now that all intended imports + # are explicit, the intuitive but potentially incompatible binding occurs due to the + # usual rules for Python submodule bindings. So for now we replace that binding with + # git.index.util, delete that, and let __getattr__ handle it and issue a warning. + # + # For the same runtime behavior, it would be enough to forgo importing util, and + # delete util as created naturally; __getattr__ would behave the same. But type + # checkers would not know what util refers to when accessed as an attribute of git. + del util + + # This is "hidden" to preserve static checking for undefined/misspelled attributes. __getattr__ = _getattr # { Initialize git executable path From bdabb21fe62063ce51fcae6b5f499f10f55525c5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 29 Mar 2024 18:52:10 -0400 Subject: [PATCH 0973/1392] Expand USE_SHELL docstring; clarify a test usage In the USE_SHELL docstring: - Restore the older wording "when executing git commands" rather than "to execute git commands". I've realized that longer phrase, which dates back to the introduction of USE_SHELL in 1c2dd54, is clearer, because readers less familiar with GitPython's overall design and operation will still not be misled into thinking USE_SHELL ever affects whether GitPython uses an external git command, versus some other mechanism, to do something. - Give some more information about why USE_SHELL = True is unsafe (though further revision or clarification may be called for). - Note some non-obvious aspects of USE_SHELL, that some of the way it interacts with other aspects of GitPython is not part of what is or has been documented about it, and in practice changes over time. The first example relates to #1792; the second may help users understand why code that enables USE_SHELL on Windows, in addition to being unsafe there, often breaks immediately on other platforms; the third is included so the warnings in the expanded docstring are not interpreted as a new commitment that any shell syntax that may have a *desired* effect in some application will continue to have the same effect in the future. - Cover a second application that might lead, or have led, to setting USE_SHELL to True, and explain what to do instead. In test_successful_default_refresh_invalidates_cached_version_info: - Rewrite the commented explanation of a special variant of that second application, where the usual easy alternatives are not used because part of the goal of the test is to check a "default" scenario that does not include either of them. This better explains why that choice is made in the test, and also hopefully will prevent anyone from thinking that test is a model for another situation (which, as noted, is unlikely to be the case even in unit tests). --- git/cmd.py | 40 +++++++++++++++++++++++++++++++--------- test/test_git.py | 14 +++++++------- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index eed82d7dd..42e6e927c 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -405,23 +405,45 @@ def __setstate__(self, d: Dict[str, Any]) -> None: """Enables debugging of GitPython's git commands.""" USE_SHELL: bool = False - """Deprecated. If set to ``True``, a shell will be used to execute git commands. + """Deprecated. If set to ``True``, a shell will be used when executing git commands. + + Code that uses ``USE_SHELL = True`` or that passes ``shell=True`` to any GitPython + functions should be updated to use the default value of ``False`` instead. ``True`` + is unsafe unless the effect of syntax treated specially by the shell is fully + considered and accounted for, which is not possible under most circumstances. As + detailed below, it is also no longer needed, even where it had been in the past. + + In addition, how a value of ``True`` interacts with some aspects of GitPython's + operation is not precisely specified and may change without warning, even before + GitPython 4.0.0 when :attr:`USE_SHELL` may be removed. This includes: + + * Whether or how GitPython automatically customizes the shell environment. + + * Whether, outside of Windows (where :class:`subprocess.Popen` supports lists of + separate arguments even when ``shell=True``), this can be used with any GitPython + functionality other than direct calls to the :meth:`execute` method. + + * Whether any GitPython feature that runs git commands ever attempts to partially + sanitize data a shell may treat specially. Currently this is not done. Prior to GitPython 2.0.8, this had a narrow purpose in suppressing console windows in graphical Windows applications. In 2.0.8 and higher, it provides no benefit, as GitPython solves that problem more robustly and safely by using the ``CREATE_NO_WINDOW`` process creation flag on Windows. - Code that uses ``USE_SHELL = True`` or that passes ``shell=True`` to any GitPython - functions should be updated to use the default value of ``False`` instead. ``True`` - is unsafe unless the effect of shell expansions is fully considered and accounted - for, which is not possible under most circumstances. + Because Windows path search differs subtly based on whether a shell is used, in rare + cases changing this from ``True`` to ``False`` may keep an unusual git "executable", + such as a batch file, from being found. To fix this, set the command name or full + path in the :envvar:`GIT_PYTHON_GIT_EXECUTABLE` environment variable or pass the + full path to :func:`git.refresh` (or invoke the script using a ``.exe`` shim). - See: + Further reading: - - :meth:`Git.execute` (on the ``shell`` parameter). - - https://github.com/gitpython-developers/GitPython/commit/0d9390866f9ce42870d3116094cd49e0019a970a - - https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags + * :meth:`Git.execute` (on the ``shell`` parameter). + * https://github.com/gitpython-developers/GitPython/commit/0d9390866f9ce42870d3116094cd49e0019a970a + * https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags + * https://github.com/python/cpython/issues/91558#issuecomment-1100942950 + * https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessw """ _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" diff --git a/test/test_git.py b/test/test_git.py index 112e2f0eb..94e68ecf0 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -669,13 +669,13 @@ def test_successful_default_refresh_invalidates_cached_version_info(self): stack.enter_context(_patch_out_env("GIT_PYTHON_GIT_EXECUTABLE")) if sys.platform == "win32": - # On Windows, use a shell so "git" finds "git.cmd". (In the infrequent - # case that this effect is desired in production code, it should not be - # done with this technique. USE_SHELL=True is less secure and reliable, - # as unintended shell expansions can occur, and is deprecated. Instead, - # use a custom command, by setting the GIT_PYTHON_GIT_EXECUTABLE - # environment variable to git.cmd or by passing git.cmd's full path to - # git.refresh. Or wrap the script with a .exe shim.) + # On Windows, use a shell so "git" finds "git.cmd". The correct and safe + # ways to do this straightforwardly are to set GIT_PYTHON_GIT_EXECUTABLE + # to git.cmd in the environment, or call git.refresh with the command's + # full path. See the Git.USE_SHELL docstring for deprecation details. + # But this tests a "default" scenario where neither is done. The + # approach used here, setting USE_SHELL to True so PATHEXT is honored, + # should not be used in production code (nor even in most test cases). stack.enter_context(mock.patch.object(Git, "USE_SHELL", True)) new_git = Git() From b51b08052821f91a61a40808328aed0ac35935a8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 29 Mar 2024 19:51:32 -0400 Subject: [PATCH 0974/1392] Explain the approach in test.deprecation to static checking And why this increases the importance of the warn_unused_ignored mypy configuration option. --- pyproject.toml | 2 +- test/deprecation/__init__.py | 17 +++++++++++++++++ test/deprecation/test_cmd_git.py | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5eac2be09..6cb05f96e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ files = ["git/", "test/deprecation/"] disallow_untyped_defs = true no_implicit_optional = true warn_redundant_casts = true -warn_unused_ignores = true +warn_unused_ignores = true # Useful in general, but especially in test/deprecation. warn_unreachable = true implicit_reexport = true # strict = true diff --git a/test/deprecation/__init__.py b/test/deprecation/__init__.py index 56b5d89db..fec3126d2 100644 --- a/test/deprecation/__init__.py +++ b/test/deprecation/__init__.py @@ -1,2 +1,19 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Tests of deprecation warnings and possible related attribute bugs. + +Most deprecation warnings are "basic" in the sense that there is no special complexity +to consider, in introducing them. However, to issue deprecation warnings on mere +attribute access can involve adding new dynamic behavior. This can lead to subtle bugs +or less useful dynamic metadata. It can also weaken static typing, as happens if a type +checker sees a method like ``__getattr__`` in a module or class whose attributes it did +not already judge to be dynamic. This test.deprecation submodule covers all three cases: +the basic cases, subtle dynamic behavior, and subtle static type checking issues. + +Static type checking is "tested" by a combination of code that should not be treated as +a type error but would be in the presence of particular bugs, and code that *should* be +treated as a type error and is accordingly marked ``# type: ignore[REASON]`` (for +specific ``REASON``. The latter will only produce mypy errors when the expectation is +not met if it is configured with ``warn_unused_ignores = true``. +""" diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index b15f219d8..3f87037a0 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -1,7 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Tests for static and dynamic characteristics of Git class and instance attributes. +"""Tests for dynamic and static characteristics of Git class and instance attributes. Currently this all relates to the deprecated :attr:`Git.USE_SHELL` class attribute, which can also be accessed through instances. Some tests directly verify its behavior, From 7cd3aa913077d55bbdb7ca01e6b7df593121f643 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 29 Mar 2024 22:55:36 -0400 Subject: [PATCH 0975/1392] Make test.performance.lib docstring more specific To distinguish it from the more general test.lib as well as from the forthcoming test.deprecation.lib. --- test/performance/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/performance/lib.py b/test/performance/lib.py index 2ca3c409b..c24599986 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -1,7 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Support library for tests.""" +"""Support library for performance tests.""" import logging import os From cf2576e406006ec28bcc85565a7ef85864cbd39e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 29 Mar 2024 23:15:52 -0400 Subject: [PATCH 0976/1392] Make/use test.deprecation.lib; abandon idea to filter by module This creates a module test.deprecation.lib in the test suite for a couple general helpers used in deprecation tests, one of which is now used in two of the test modules and the other of which happens only to be used in one but is concepually related to the first. The assert_no_deprecation_warning context manager function fixes two different flawed approaches to that, which were in use earlier: - In test_basic, only DeprecationWarning and not the less significant PendingDeprecationWarning had been covere, which it should, at least for symmetry, since pytest.deprecated_call() treats it like DeprecationWarning. There was also a comment expressing a plan to make it filter only for warnings originating from GitPython, which was a flawed idea, as detailed below. - In test_cmd_git, the flawed idea of attempting to filter only for warnings originating from GitPython was implemented. The problem with this is that it is heavily affected by stacklevel and its interaction with the pattern of calls through which the warning arises: warnings could actually emanate from code in GitPython's git module, but be registered as having come from test code, a callback in gitdb, smmap, or the standard library, or even the pytest test runner. Some of these are more likely than others, but all are possible especially considering the possibility of a bug in the value passed to warning.warn as stacklevel. (It may be valuable for such bugs to cause tests to fail, but they should not cause otherwise failing tests to wrongly pass.) It is probably feasible to implement such filtering successfully, but I don't think it's worthwhile for the small reduction in likelihood of failing later on an unrealted DeprecationWarning from somewhere else, especially considering that GitPython's main dependencies are so minimal. --- test/deprecation/lib.py | 27 +++++++++++++++++++++++++++ test/deprecation/test_basic.py | 26 ++++++++------------------ test/deprecation/test_cmd_git.py | 24 +++++------------------- 3 files changed, 40 insertions(+), 37 deletions(-) create mode 100644 test/deprecation/lib.py diff --git a/test/deprecation/lib.py b/test/deprecation/lib.py new file mode 100644 index 000000000..9fe623a3a --- /dev/null +++ b/test/deprecation/lib.py @@ -0,0 +1,27 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Support library for deprecation tests.""" + +__all__ = ["assert_no_deprecation_warning", "suppress_deprecation_warning"] + +import contextlib +import warnings + +from typing import Generator + + +@contextlib.contextmanager +def assert_no_deprecation_warning() -> Generator[None, None, None]: + """Context manager to assert that code does not issue any deprecation warnings.""" + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + warnings.simplefilter("error", PendingDeprecationWarning) + yield + + +@contextlib.contextmanager +def suppress_deprecation_warning() -> Generator[None, None, None]: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + yield diff --git a/test/deprecation/test_basic.py b/test/deprecation/test_basic.py index 8ee7e72b1..6235a836c 100644 --- a/test/deprecation/test_basic.py +++ b/test/deprecation/test_basic.py @@ -15,9 +15,6 @@ It is inapplicable to the deprecations whose warnings are tested in this module. """ -import contextlib -import warnings - import pytest from git.diff import NULL_TREE @@ -25,6 +22,8 @@ from git.repo import Repo from git.util import Iterable as _Iterable, IterableObj +from .lib import assert_no_deprecation_warning + # typing ----------------------------------------------------------------- from typing import Generator, TYPE_CHECKING @@ -38,15 +37,6 @@ # ------------------------------------------------------------------------ -@contextlib.contextmanager -def _assert_no_deprecation_warning() -> Generator[None, None, None]: - """Context manager to assert that code does not issue any deprecation warnings.""" - with warnings.catch_warnings(): - # FIXME: Refine this to filter for deprecation warnings from GitPython. - warnings.simplefilter("error", DeprecationWarning) - yield - - @pytest.fixture def commit(tmp_path: "Path") -> Generator["Commit", None, None]: """Fixture to supply a one-commit repo's commit, enough for deprecation tests.""" @@ -72,7 +62,7 @@ def test_diff_renamed_warns(diff: "Diff") -> None: def test_diff_renamed_file_does_not_warn(diff: "Diff") -> None: """The preferred Diff.renamed_file property issues no deprecation warning.""" - with _assert_no_deprecation_warning(): + with assert_no_deprecation_warning(): diff.renamed_file @@ -84,13 +74,13 @@ def test_commit_trailers_warns(commit: "Commit") -> None: def test_commit_trailers_list_does_not_warn(commit: "Commit") -> None: """The nondeprecated Commit.trailers_list property issues no deprecation warning.""" - with _assert_no_deprecation_warning(): + with assert_no_deprecation_warning(): commit.trailers_list def test_commit_trailers_dict_does_not_warn(commit: "Commit") -> None: """The nondeprecated Commit.trailers_dict property issues no deprecation warning.""" - with _assert_no_deprecation_warning(): + with assert_no_deprecation_warning(): commit.trailers_dict @@ -102,7 +92,7 @@ def test_traverse_list_traverse_in_base_class_warns(commit: "Commit") -> None: def test_traversable_list_traverse_override_does_not_warn(commit: "Commit") -> None: """Calling list_traverse on concrete subclasses is not deprecated, does not warn.""" - with _assert_no_deprecation_warning(): + with assert_no_deprecation_warning(): commit.list_traverse() @@ -114,7 +104,7 @@ def test_traverse_traverse_in_base_class_warns(commit: "Commit") -> None: def test_traverse_traverse_override_does_not_warn(commit: "Commit") -> None: """Calling traverse on concrete subclasses is not deprecated, does not warn.""" - with _assert_no_deprecation_warning(): + with assert_no_deprecation_warning(): commit.traverse() @@ -128,7 +118,7 @@ class Derived(_Iterable): def test_iterable_obj_inheriting_does_not_warn() -> None: """Subclassing git.util.IterableObj is not deprecated, does not warn.""" - with _assert_no_deprecation_warning(): + with assert_no_deprecation_warning(): class Derived(IterableObj): pass diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index 3f87037a0..6c592454a 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -56,12 +56,10 @@ metaclass marginal enough, to justify introducing a metaclass to issue the warnings. """ -import contextlib import logging import sys from typing import Generator import unittest.mock -import warnings if sys.version_info >= (3, 11): from typing import assert_type @@ -73,6 +71,8 @@ from git.cmd import Git +from .lib import assert_no_deprecation_warning, suppress_deprecation_warning + _USE_SHELL_DEPRECATED_FRAGMENT = "Git.USE_SHELL is deprecated" """Text contained in all USE_SHELL deprecation warnings, and starting most of them.""" @@ -82,13 +82,6 @@ _logger = logging.getLogger(__name__) -@contextlib.contextmanager -def _suppress_deprecation_warning() -> Generator[None, None, None]: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - yield - - @pytest.fixture def restore_use_shell_state() -> Generator[None, None, None]: """Fixture to attempt to restore state associated with the USE_SHELL attribute. @@ -129,7 +122,7 @@ def restore_use_shell_state() -> Generator[None, None, None]: # Try to save the original public value. Rather than attempt to restore a state # where the attribute is not set, if we cannot do this we allow AttributeError # to propagate out of the fixture, erroring the test case before its code runs. - with _suppress_deprecation_warning(): + with suppress_deprecation_warning(): old_public_value = Git.USE_SHELL # This doesn't have its own try-finally because pytest catches exceptions raised @@ -137,7 +130,7 @@ def restore_use_shell_state() -> Generator[None, None, None]: yield # Try to restore the original public value. - with _suppress_deprecation_warning(): + with suppress_deprecation_warning(): Git.USE_SHELL = old_public_value finally: # Try to restore the original private state. @@ -323,14 +316,7 @@ def test_execute_without_shell_arg_does_not_warn() -> None: USE_SHELL is to be used, the way Git.execute itself accesses USE_SHELL does not issue a deprecation warning. """ - with warnings.catch_warnings(): - for category in DeprecationWarning, PendingDeprecationWarning: - warnings.filterwarnings( - action="error", - category=category, - module=r"git(?:\.|$)", - ) - + with assert_no_deprecation_warning(): Git().version() From c7675d2cedcd737f20359a4a786e213510452413 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 30 Mar 2024 08:56:41 +0100 Subject: [PATCH 0977/1392] update security policy, to use GitHub instead of email --- SECURITY.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index cbfaafdde..d39425b70 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,5 +11,4 @@ Only the latest version of GitPython can receive security updates. If a vulnerab ## Reporting a Vulnerability -Please report private portions of a vulnerability to sebastian.thiel@icloud.com that would help to reproduce and fix it. To receive updates on progress and provide -general information to the public, you can create an issue [on the issue tracker](https://github.com/gitpython-developers/GitPython/issues). +Please report private portions of a vulnerability to . Doing so helps to receive updates and collaborate on the matter, without disclosing it publicliy right away. From f92f4c3bf902c7fc3887cfd969b3e54f581f18f8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 30 Mar 2024 20:16:45 -0400 Subject: [PATCH 0978/1392] Clarify security risk in USE_SHELL doc and warnings --- git/cmd.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 42e6e927c..b2829801f 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -314,10 +314,10 @@ def dict_to_slots_and__excluded_are_none(self: object, d: Mapping[str, Any], exc ) _USE_SHELL_DANGER_MESSAGE = ( - "Setting Git.USE_SHELL to True is unsafe and insecure, and should be avoided, " - "because the effect of shell metacharacters and shell expansions cannot usually be " - "accounted for. In addition, Git.USE_SHELL is deprecated and will be removed in a " - "future release." + "Setting Git.USE_SHELL to True is unsafe and insecure, as the effect of special " + "shell syntax cannot usually be accounted for. This can result in a command " + "injection vulnerability and arbitrary code execution. Git.USE_SHELL is deprecated " + "and will be removed in a future release." ) @@ -413,6 +413,13 @@ def __setstate__(self, d: Dict[str, Any]) -> None: considered and accounted for, which is not possible under most circumstances. As detailed below, it is also no longer needed, even where it had been in the past. + It is in many if not most cases a command injection vulnerability for an application + to set :attr:`USE_SHELL` to ``True``. Any attacker who can cause a specially crafted + fragment of text to make its way into any part of any argument to any git command + (including paths, branch names, etc.) can cause the shell to read and write + arbitrary files and execute arbitrary commands. Innocent input may also accidentally + contain special shell syntax, leading to inadvertent malfunctions. + In addition, how a value of ``True`` interacts with some aspects of GitPython's operation is not precisely specified and may change without warning, even before GitPython 4.0.0 when :attr:`USE_SHELL` may be removed. This includes: From 8327b458bdcf73895aa3b0a9a990a61ce2e54ee9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 16:39:20 -0400 Subject: [PATCH 0979/1392] Test GitMeta alias --- test/deprecation/test_cmd_git.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/deprecation/test_cmd_git.py b/test/deprecation/test_cmd_git.py index 6c592454a..e44490273 100644 --- a/test/deprecation/test_cmd_git.py +++ b/test/deprecation/test_cmd_git.py @@ -69,7 +69,7 @@ import pytest from pytest import WarningsRecorder -from git.cmd import Git +from git.cmd import Git, GitMeta from .lib import assert_no_deprecation_warning, suppress_deprecation_warning @@ -377,3 +377,15 @@ def test_instance_dir() -> None: instance = Git() actual = set(dir(instance)) assert _EXPECTED_DIR_SUBSET <= actual + + +def test_metaclass_alias() -> None: + """GitMeta aliases Git's metaclass, whether that is type or a custom metaclass.""" + + def accept_metaclass_instance(cls: GitMeta) -> None: + """Check that cls is statically recognizable as an instance of GitMeta.""" + + accept_metaclass_instance(Git) # assert_type would expect Type[Git], not GitMeta. + + # This comes after the static check, just in case it would affect the inference. + assert type(Git) is GitMeta From f6060df576acda613227a57f03c01d235eceaeae Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 27 Mar 2024 17:38:31 -0400 Subject: [PATCH 0980/1392] Add GitMeta alias Hopefully no one will ever need it. --- git/cmd.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index b2829801f..90fc39cd6 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -5,7 +5,7 @@ from __future__ import annotations -__all__ = ["Git"] +__all__ = ["GitMeta", "Git"] import contextlib import io @@ -354,6 +354,32 @@ def __setattr(cls, name: str, value: Any) -> Any: __setattr__ = __setattr +GitMeta = _GitMeta +"""Alias of :class:`Git`'s metaclass, whether it is :class:`type` or a custom metaclass. + +Whether the :class:`Git` class has the default :class:`type` as its metaclass or uses a +custom metaclass is not documented and may change at any time. This statically checkable +metaclass alias is equivalent at runtime to ``type(Git)``. This should almost never be +used. Code that benefits from it is likely to be remain brittle even if it is used. + +In view of the :class:`Git` class's intended use and :class:`Git` objects' dynamic +callable attributes representing git subcommands, it rarely makes sense to inherit from +:class:`Git` at all. Using :class:`Git` in multiple inheritance can be especially tricky +to do correctly. Attempting uses of :class:`Git` where its metaclass is relevant, such +as when a sibling class has an unrelated metaclass and a shared lower bound metaclass +might have to be introduced to solve a metaclass conflict, is not recommended. + +:note: + The correct static type of the :class:`Git` class itself, and any subclasses, is + ``Type[Git]``. (This can be written as ``type[Git]`` in Python 3.9 later.) + + :class:`GitMeta` should never be used in any annotation where ``Type[Git]`` is + intended or otherwise possible to use. This alias is truly only for very rare and + inherently precarious situations where it is necessary to deal with the metaclass + explicitly. +""" + + class Git(metaclass=_GitMeta): """The Git class manages communication with the Git binary. From 53640535cf8314366a01da081947dd8504a299cd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 31 Mar 2024 10:05:27 +0200 Subject: [PATCH 0981/1392] bump version to 3.1.43 --- VERSION | 2 +- doc/source/changes.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f69b7f25e..d1bf6638d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.42 +3.1.43 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index c75e5eded..0bc757134 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,15 @@ Changelog ========= +3.1.43 +====== + +A major visible change will be the added deprecation- or user-warnings, +and greatly improved typing. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.43 + 3.1.42 ====== From 83bed19f360b69dad26b7d9b00ffd837c8075b7a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 31 Mar 2024 14:57:52 -0400 Subject: [PATCH 0982/1392] Fix wording of comment about the /cygdrive prefix --- git/repo/fun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/repo/fun.py b/git/repo/fun.py index e44d9c644..182cf82ed 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -112,7 +112,7 @@ def find_submodule_git_dir(d: PathLike) -> Optional[PathLike]: path = content[8:] if Git.is_cygwin(): - # Cygwin creates submodules prefixed with `/cygdrive/...` suffixes. + # Cygwin creates submodules prefixed with `/cygdrive/...`. # Cygwin git understands Cygwin paths much better than Windows ones. # Also the Cygwin tests are assuming Cygwin paths. path = cygpath(path) From 988d97bf12c5b15ff4693a5893134271dd8d8a28 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 31 Mar 2024 15:40:47 -0400 Subject: [PATCH 0983/1392] Fix typo in _get_exe_extensions PATHEXT fallback PATHEXT lists file extensions with the ".". In the fallback given in _get_exe_extensions, the other extensions had this, but ".COM" was listed without the ".". This fixes that. This is very minor because _get_exe_extensions is nonpublic and not currently used on native Windows, which is the platform where the PATHEXT fallback code would be used. Specifically, _get_exe_extensions is called only in py_where, which while named with no leading underscore is nonpublic do not being (and never having been) listed in __all__. As its docstring states, it is an implementation detail of is_cygwin_git and not intended for any other use. More specifically, is_cygwin_git currently immediately returns False on *native* Windows (even if the git executable GitPython is using is a Cygwin git executable). Only on Cygwin, or other systems that are not native Windows, does it try to check the git executable (by calling its _is_cygwin_git helper, which uses py_where). --- git/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index 8c1c26012..11f963e02 100644 --- a/git/util.py +++ b/git/util.py @@ -339,7 +339,7 @@ def _get_exe_extensions() -> Sequence[str]: if PATHEXT: return tuple(p.upper() for p in PATHEXT.split(os.pathsep)) elif sys.platform == "win32": - return (".BAT", "COM", ".EXE") + return (".BAT", ".COM", ".EXE") else: return () From f18df8edcf5de5971e4dd01f15ad32411e38244e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 31 Mar 2024 18:07:32 -0400 Subject: [PATCH 0984/1392] Don't pass --disable-warnings to pytest --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6cb05f96e..ee54edb78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] -addopts = "--cov=git --cov-report=term --disable-warnings -ra" +addopts = "--cov=git --cov-report=term -ra" filterwarnings = "ignore::DeprecationWarning" python_files = "test_*.py" tmp_path_retention_policy = "failed" @@ -13,7 +13,6 @@ testpaths = "test" # Space separated list of paths from root e.g test tests doc # --cov-report term-missing # to terminal with line numbers # --cov-report html:path # html file at path # --maxfail # number of errors before giving up -# -disable-warnings # Disable pytest warnings (not codebase warnings) # -rfE # default test summary: list fail and error # -ra # test summary: list all non-passing (fail, error, skip, xfail, xpass) # --ignore-glob=**/gitdb/* # ignore glob paths From 0152b528a035566364fc8a0e1a80829c6d495301 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 1 Apr 2024 14:53:43 -0400 Subject: [PATCH 0985/1392] Update the comment about `--mixed` and paths This updates the comment in HEAD.reset about why `--mixed` is omitted from the git command constructed to perform a reset where paths are being passed, adding specific information about the git versions where this is deprecated, and changing the more-info link from an old GitPython issue that is no longer retrievable to #1876. --- git/refs/head.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/refs/head.py b/git/refs/head.py index 7e6fc3377..683634451 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -99,8 +99,8 @@ def reset( if index: mode = "--mixed" - # It appears some git versions declare mixed and paths deprecated. - # See http://github.com/Byron/GitPython/issues#issue/2. + # Explicit "--mixed" when passing paths is deprecated since git 1.5.4. + # See https://github.com/gitpython-developers/GitPython/discussions/1876. if paths: mode = None # END special case From a9593c7c56e76bdef35245be00bf23abdc3ba4c0 Mon Sep 17 00:00:00 2001 From: Eduard Talanov <89387701+EduardTalanov@users.noreply.github.com> Date: Fri, 5 Apr 2024 10:10:53 +0200 Subject: [PATCH 0986/1392] Update remote.py Fixed the error of updating shallow submodules --- git/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/remote.py b/git/remote.py index f2ecd0f36..37c991d27 100644 --- a/git/remote.py +++ b/git/remote.py @@ -316,7 +316,7 @@ class FetchInfo(IterableObj): ERROR, ) = [1 << x for x in range(8)] - _re_fetch_result = re.compile(r"^ *(.) (\[[\w \.$@]+\]|[\w\.$@]+) +(.+) -> ([^ ]+)( \(.*\)?$)?") + _re_fetch_result = re.compile(r"^ *(?:.{0,3})(.) (\[[\w \.$@]+\]|[\w\.$@]+) +(.+) -> ([^ ]+)( \(.*\)?$)?") _flag_map: Dict[flagKeyLiteral, int] = { "!": ERROR, From 55c30a34185e13c31ab14c39f7a7dd0db3a494e4 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 11 Apr 2024 19:55:10 -0400 Subject: [PATCH 0987/1392] OSS-Fuzz test initial migration Migrates the OSS-Fuzz tests and setup scripts from the OSS-Fuzz repository to GitPython's repo as discussed here: https://github.com/gitpython-developers/GitPython/issues/1887#issuecomment-2028599381 These files include the changes that were originally proposed in: https://github.com/google/oss-fuzz/pull/11763 Additional changes include: - A first pass at documenting the contents of the fuzzing set up in a dedicated README.md - Adding the dictionary files to this repo for improved visibility. Seed corpra zips are still located in an external repo pending further discussion regarding where those should live in the long term. --- fuzzing/README.md | 172 ++++++++++++++++++ fuzzing/dictionaries/fuzz_config.dict | 56 ++++++ fuzzing/dictionaries/fuzz_tree.dict | 13 ++ fuzzing/fuzz-targets/fuzz_config.py | 51 ++++++ fuzzing/fuzz-targets/fuzz_tree.py | 59 ++++++ fuzzing/oss-fuzz-scripts/build.sh | 37 ++++ .../container-environment-bootstrap.sh | 56 ++++++ 7 files changed, 444 insertions(+) create mode 100644 fuzzing/README.md create mode 100644 fuzzing/dictionaries/fuzz_config.dict create mode 100644 fuzzing/dictionaries/fuzz_tree.dict create mode 100644 fuzzing/fuzz-targets/fuzz_config.py create mode 100644 fuzzing/fuzz-targets/fuzz_tree.py create mode 100644 fuzzing/oss-fuzz-scripts/build.sh create mode 100644 fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh diff --git a/fuzzing/README.md b/fuzzing/README.md new file mode 100644 index 000000000..6853c5002 --- /dev/null +++ b/fuzzing/README.md @@ -0,0 +1,172 @@ +# Fuzzing GitPython + +[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/gitpython.svg)][oss-fuzz-issue-tracker] + +This directory contains files related to GitPython's suite of fuzz tests that are executed daily on automated +infrastructure provided by [OSS-Fuzz][oss-fuzz-repo]. This document aims to provide necessary information for working +with fuzzing in GitPython. + +The details about the latest OSS-Fuzz test status, including build logs and coverage reports, is made available +at [this link](https://introspector.oss-fuzz.com/project-profile?project=gitpython). + +## How to Contribute + +There are many ways to contribute to GitPython's fuzzing efforts! Contributions are welcomed through issues, +discussions, or pull requests on this repository. + +Areas that are particularly appreciated include: + +- **Tackling the existing backlog of open issues**. While fuzzing is an effective way to identify bugs, that information + isn't useful unless they are fixed. If you are not sure where to start, the issues tab is a great place to get ideas! +- **Improvements to this (or other) documentation** make it easier for new contributors to get involved, so even small + improvements can have a large impact over time. If you see something that could be made easier by a documentation + update of any size, please consider suggesting it! + +For everything else, such as expanding test coverage, optimizing test performance, or enhancing error detection +capabilities, jump in to the "Getting Started" section below. + +## Getting Started with Fuzzing GitPython + +> [!TIP] +> **New to fuzzing or unfamiliar with OSS-Fuzz?** +> +> These resources are an excellent place to start: +> +> - [OSS-Fuzz documentation][oss-fuzz-docs] - Continuous fuzzing service for open source software. +> - [Google/fuzzing][google-fuzzing-repo] - Tutorials, examples, discussions, research proposals, and other resources + related to fuzzing. +> - [CNCF Fuzzing Handbook](https://github.com/cncf/tag-security/blob/main/security-fuzzing-handbook/handbook-fuzzing.pdf) - + A comprehensive guide for fuzzing open source software. +> - [Efficient Fuzzing Guide by The Chromium Project](https://chromium.googlesource.com/chromium/src/+/main/testing/libfuzzer/efficient_fuzzing.md) - + Explores strategies to enhance the effectiveness of your fuzz tests, recommended for those looking to optimize their + testing efforts. + +### Setting Up Your Local Environment + +Before contributing to fuzzing efforts, ensure Python and Docker are installed on your machine. Docker is required for +running fuzzers in containers provided by OSS-Fuzz. [Install Docker](https://docs.docker.com/get-docker/) following the +official guide if you do not already have it. + +### Understanding Existing Fuzz Targets + +Review the `fuzz-targets/` directory to familiarize yourself with how existing tests are implemented. See +the [Files & Directories Overview](#files--directories-overview) for more details on the directory structure. + +### Contributing to Fuzz Tests + +Start by reviewing the [Atheris documentation][atheris-repo] and the section +on [Running Fuzzers Locally](#running-fuzzers-locally) to begin writing or improving fuzz tests. + +## Files & Directories Overview + +The `fuzzing/` directory is organized into three key areas: + +### Fuzz Targets (`fuzz-targets/`) + +Contains Python files for each fuzz test, targeting specific functionalities of GitPython. + +**Things to Know**: + +- Each fuzz test targets a specific part of GitPython's functionality. +- Test files adhere to the naming convention: `fuzz_.py`, where `` indicates the + functionality targeted by the test. +- Any functionality that involves performing operations on input data is a possible candidate for fuzz testing, but + features that involve processing untrusted user input or parsing operations are typically going to be the most + interesting. +- The goal of these tests is to identify previously unknown or unexpected error cases caused by a given input. For that + reason, fuzz tests should gracefully handle anticipated exception cases with a `try`/`except` block to avoid false + positives that halt the fuzzing engine. + +### Dictionaries (`dictionaries/`) + +Provides hints to the fuzzing engine about inputs that might trigger unique code paths. Each fuzz target may have a +corresponding `.dict` file. For details on how these are used, refer +to [LibFuzzer documentation](https://llvm.org/docs/LibFuzzer.html#dictionaries). + +**Things to Know**: + +- OSS-Fuzz loads dictionary files per fuzz target if one exists with the same name, all others are ignored. +- Most entries in the dictionary files found here are escaped hex or Unicode values that were recommended by the fuzzing + engine after previous runs. +- A default set of dictionary entries are created for all fuzz targets as part of the build process, regardless of an + existing file here. +- Development or updates to dictionaries should reflect the varied formats and edge cases relevant to the + functionalities under test. +- Example dictionaries (some of which are used to build the default dictionaries mentioned above) are can be found here: + - [AFL++ dictionary repository](https://github.com/AFLplusplus/AFLplusplus/tree/stable/dictionaries#readme) + - [Google/fuzzing dictionary repository](https://github.com/google/fuzzing/tree/master/dictionaries) + +### OSS-Fuzz Scripts (`oss-fuzz-scripts/`) + +Includes scripts for building and integrating fuzz targets with OSS-Fuzz: + +- **`container-environment-bootstrap.sh`** - Sets up the execution environment. It is responsible for fetching default + dictionary entries and ensuring all required build dependencies are installed and up-to-date. +- **`build.sh`** - Executed within the Docker container, this script builds fuzz targets with necessary instrumentation + and prepares seed corpora and dictionaries for use. + +## Running Fuzzers Locally + +### Direct Execution of Fuzz Targets + +For quick testing of changes, [Atheris][atheris-repo] makes it possible to execute a fuzz target directly: + +1. Install Atheris following the [installation guide][atheris-repo] for your operating system. +2. Execute a fuzz target, for example: + +```shell +python fuzzing/fuzz-targets/fuzz_config.py +``` + +### Running OSS-Fuzz Locally + +This approach uses Docker images provided by OSS-Fuzz for building and running fuzz tests locally. It offers +comprehensive features but requires a local clone of the OSS-Fuzz repository and sufficient disk space for Docker +containers. + +#### Preparation + +Set environment variables to simplify command usage: + +```shell +export SANITIZER=address # Can be either 'address' or 'undefined'. +export FUZZ_TARGET=fuzz_config # specify the fuzz target without the .py extension. +``` + +#### Build and Run + +Clone the OSS-Fuzz repository and prepare the Docker environment: + +```shell +git clone --depth 1 https://github.com/google/oss-fuzz.git oss-fuzz +cd oss-fuzz +python infra/helper.py build_image gitpython +python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython +``` + +Verify the build of your fuzzers with the optional `check_build` command: + +```shell +python infra/helper.py check_build gitpython +``` + +Execute the desired fuzz target: + +```shell +python infra/helper.py run_fuzzer gitpython $FUZZ_TARGET +``` + +#### Next Steps + +For detailed instructions on advanced features like reproducing OSS-Fuzz issues or using the Fuzz Introspector, refer +to [the official OSS-Fuzz documentation][oss-fuzz-docs]. + +[oss-fuzz-repo]: https://github.com/google/oss-fuzz + +[oss-fuzz-docs]: https://google.github.io/oss-fuzz + +[oss-fuzz-issue-tracker]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:gitpython + +[google-fuzzing-repo]: https://github.com/google/fuzzing + +[atheris-repo]: https://github.com/google/atheris diff --git a/fuzzing/dictionaries/fuzz_config.dict b/fuzzing/dictionaries/fuzz_config.dict new file mode 100644 index 000000000..b545ddfc8 --- /dev/null +++ b/fuzzing/dictionaries/fuzz_config.dict @@ -0,0 +1,56 @@ +"\\004\\000\\000\\000\\000\\000\\000\\000" +"\\006\\000\\000\\000\\000\\000\\000\\000" +"_validate_value_" +"\\000\\000\\000\\000\\000\\000\\000\\000" +"rem" +"__eq__" +"\\001\\000\\000\\000" +"__abstrac" +"_mutating_methods_" +"items" +"\\0021\\"" +"\\001\\000" +"\\000\\000\\000\\000" +"DEFAULT" +"getfloat" +"\\004\\000\\000\\000\\000\\000\\000\\000" +"news" +"\\037\\000\\000\\000\\000\\000\\000\\000" +"\\001\\000\\000\\000\\000\\000\\000\\037" +"\\000\\000\\000\\000\\000\\000\\000\\014" +"list" +"\\376\\377\\377\\377\\377\\377\\377\\377" +"items_all" +"\\004\\000\\000\\000\\000\\000\\000\\000" +"\\377\\377\\377\\377\\377\\377\\377\\014" +"\\001\\000\\000\\000" +"_acqui" +"\\000\\000\\000\\000\\000\\000\\000\\000" +"__ne__" +"__exit__" +"__modu" +"uucp" +"__str__" +"\\001\\000\\000\\000" +"\\017\\000\\000\\000\\000\\000\\000\\000" +"_has_incl" +"update" +"\\377\\377\\377\\377\\377\\377\\377\\023" +"setdef" +"setdefaul" +"\\000\\000\\000\\000" +"\\001\\000\\000\\000" +"\\001\\000" +"\\022\\000\\000\\000\\000\\000\\000\\000" +"_value_to_string" +"__abstr" +"\\001\\000\\000\\000\\000\\000\\000\\000" +"\\000\\000\\000\\000\\000\\000\\000\\022" +"\\377\\377\\377\\377" +"\\004\\000\\000\\000\\000\\000\\000\\000" +"\\000\\000\\000\\000\\000\\000\\000\\000" +"\\000\\000\\000\\000\\000\\000\\000\\037" +"\\001\\000\\000\\000\\000\\000\\000\\013" +"_OPT_TM" +"__name__" +"_get_conv" diff --git a/fuzzing/dictionaries/fuzz_tree.dict b/fuzzing/dictionaries/fuzz_tree.dict new file mode 100644 index 000000000..3ebe52b7f --- /dev/null +++ b/fuzzing/dictionaries/fuzz_tree.dict @@ -0,0 +1,13 @@ +"\\001\\000\\000\\000" +"_join_multiline_va" +"setdef" +"1\\000\\000\\000\\000\\000\\000\\000" +"\\000\\000\\000\\000\\000\\000\\000\\020" +"\\377\\377\\377\\377\\377\\377\\377r" +"\\001\\000\\000\\000\\000\\000\\000\\001" +"\\000\\000\\000\\000\\000\\000\\000\\014" +"\\000\\000\\000\\000\\000\\000\\000\\003" +"\\001\\000" +"\\032\\000\\000\\000\\000\\000\\000\\000" +"-\\000\\000\\000\\000\\000\\000\\000" +"__format" diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py new file mode 100644 index 000000000..1403c96e4 --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_config.py @@ -0,0 +1,51 @@ +#!/usr/bin/python3 +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import atheris +import sys +import io +from configparser import MissingSectionHeaderError, ParsingError + +with atheris.instrument_imports(): + from git import GitConfigParser + + +def TestOneInput(data): + sio = io.BytesIO(data) + sio.name = "/tmp/fuzzconfig.config" + git_config = GitConfigParser(sio) + try: + git_config.read() + except (MissingSectionHeaderError, ParsingError, UnicodeDecodeError): + return -1 # Reject inputs raising expected exceptions + except (IndexError, ValueError) as e: + if isinstance(e, IndexError) and "string index out of range" in str(e): + # Known possibility that might be patched + # See: https://github.com/gitpython-developers/GitPython/issues/1887 + pass + elif isinstance(e, ValueError) and "embedded null byte" in str(e): + # The `os.path.expanduser` function, which does not accept strings + # containing null bytes might raise this. + return -1 + else: + raise e # Raise unanticipated exceptions as they might be bugs + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/fuzz-targets/fuzz_tree.py b/fuzzing/fuzz-targets/fuzz_tree.py new file mode 100644 index 000000000..53258fb1e --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_tree.py @@ -0,0 +1,59 @@ +#!/usr/bin/python3 +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import atheris +import io +import sys +import os +import shutil + +with atheris.instrument_imports(): + from git.objects import Tree + from git.repo import Repo + + +def TestOneInput(data): + fdp = atheris.FuzzedDataProvider(data) + git_dir = "/tmp/.git" + head_file = os.path.join(git_dir, "HEAD") + refs_dir = os.path.join(git_dir, "refs") + common_dir = os.path.join(git_dir, "commondir") + objects_dir = os.path.join(git_dir, "objects") + + if os.path.isdir(git_dir): + shutil.rmtree(git_dir) + + os.mkdir(git_dir) + with open(head_file, "w") as f: + f.write(fdp.ConsumeUnicodeNoSurrogates(1024)) + os.mkdir(refs_dir) + os.mkdir(common_dir) + os.mkdir(objects_dir) + + _repo = Repo("/tmp/") + + fuzz_tree = Tree(_repo, Tree.NULL_BIN_SHA, 0, "") + try: + fuzz_tree._deserialize(io.BytesIO(data)) + except IndexError: + return -1 + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh new file mode 100644 index 000000000..fdab7a1e0 --- /dev/null +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -euo pipefail + +python3 -m pip install . + +# Directory to look in for dictionaries, options files, and seed corpa: +SEED_DATA_DIR="$SRC/seed_data" + +find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name '*.dict' \) \ + ! \( -name '__base.*' \) -exec printf 'Copying: %s\n' {} \; \ + -exec chmod a-x {} \; \ + -exec cp {} "$OUT" \; + +# Build fuzzers in $OUT. +find "$SRC" -name 'fuzz_*.py' -print0 | while IFS= read -r -d $'\0' fuzz_harness; do + compile_python_fuzzer "$fuzz_harness" + + common_base_dictionary_filename="$SEED_DATA_DIR/__base.dict" + if [[ -r "$common_base_dictionary_filename" ]]; then + # Strip the `.py` extension from the filename and replace it with `.dict`. + fuzz_harness_dictionary_filename="$(basename "$fuzz_harness" .py).dict" + output_file="$OUT/$fuzz_harness_dictionary_filename" + + printf 'Appending %s to %s\n' "$common_base_dictionary_filename" "$output_file" + if [[ -s "$output_file" ]]; then + # If a dictionary file for this fuzzer already exists and is not empty, + # we append a new line to the end of it before appending any new entries. + # + # libfuzzer will happily ignore multiple empty lines in a dictionary but crash + # if any single line has incorrect syntax (e.g., if we accidentally add two entries to the same line.) + # See docs for valid syntax: https://llvm.org/docs/LibFuzzer.html#id32 + echo >>"$output_file" + fi + cat "$common_base_dictionary_filename" >>"$output_file" + fi +done diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh new file mode 100644 index 000000000..43c21a8da --- /dev/null +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +################# +# Prerequisites # +################# + +for cmd in python3 git wget rsync; do + command -v "$cmd" >/dev/null 2>&1 || { + printf '[%s] Required command %s not found, exiting.\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$cmd" >&2 + exit 1 + } +done + +SEED_DATA_DIR="$SRC/seed_data" +mkdir -p "$SEED_DATA_DIR" + +############# +# Functions # +############# + +download_and_concatenate_common_dictionaries() { + # Assign the first argument as the target file where all contents will be concatenated + target_file="$1" + + # Shift the arguments so the first argument (target_file path) is removed + # and only URLs are left for the loop below. + shift + + for url in "$@"; do + wget -qO- "$url" >>"$target_file" + # Ensure there's a newline between each file's content + echo >>"$target_file" + done +} + +fetch_seed_corpra() { + # Seed corpus zip files are hosted in a separate repository to avoid additional bloat in this repo. + git clone --depth 1 https://github.com/DaveLak/oss-fuzz-inputs.git oss-fuzz-inputs && + rsync -avc oss-fuzz-inputs/gitpython/corpra/ "$SEED_DATA_DIR/" && + rm -rf oss-fuzz-inputs; # Clean up the cloned repo to keep the Docker image as slim as possible. +} + +######################## +# Main execution logic # +######################## + +fetch_seed_corpra; + +download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \ + "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \ + "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict"; + +# The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below. +python3 -m pip install --upgrade pip; +python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0'; # Uses the latest versions know to work at the time of this commit. From d0c6ee62ad64a074ca885c0c2edbbc5074542b6e Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 11 Apr 2024 20:11:34 -0400 Subject: [PATCH 0988/1392] Update documentation to include fuzzing specific info As per discussion in https://github.com/gitpython-developers/GitPython/discussions/1889 --- CONTRIBUTING.md | 5 +++++ README.md | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e108f1b80..8536d7f73 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,3 +8,8 @@ The following is a short step-by-step rundown of what one typically would do to - Try to avoid massive commits and prefer to take small steps, with one commit for each. - Feel free to add yourself to AUTHORS file. - Create a pull request. + +## Fuzzing Test Specific Documentation + +For details related to contributing to the fuzzing test suite and OSS-Fuzz integration, please +refer to the dedicated [fuzzing README](./fuzzing/README.md). diff --git a/README.md b/README.md index 9bedaaae7..39f5496dc 100644 --- a/README.md +++ b/README.md @@ -240,5 +240,17 @@ Please have a look at the [contributions file][contributing]. [3-Clause BSD License](https://opensource.org/license/bsd-3-clause/), also known as the New BSD License. See the [LICENSE file][license]. +> [!NOTE] +> There are two special case files located in the `fuzzzing/` directory that are licensed differently: +> +> `fuzz_config.py` and `fuzz_tree.py` were migrated here from the OSS-Fuzz project repository where they were initially +> created and retain the original licence and copyright notice (Apache License, Version 2.0 and Copyright 2023 Google +> LLC respectively.) +> +> - **These files do not impact the licence under which GitPython releases or source code are distributed.** +> - The files located in the `fuzzzing/` directory are part of the project test suite and neither packaged nor distributed as + part of any release. + + [contributing]: https://github.com/gitpython-developers/GitPython/blob/main/CONTRIBUTING.md [license]: https://github.com/gitpython-developers/GitPython/blob/main/LICENSE From 1bc9a1a8250aa7291255cf389be2fa871c9049db Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 11 Apr 2024 21:19:24 -0400 Subject: [PATCH 0989/1392] Improve fuzzing README Adds additional documentation links and fixes some typos. --- fuzzing/README.md | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/fuzzing/README.md b/fuzzing/README.md index 6853c5002..97a62724a 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -6,7 +6,7 @@ This directory contains files related to GitPython's suite of fuzz tests that ar infrastructure provided by [OSS-Fuzz][oss-fuzz-repo]. This document aims to provide necessary information for working with fuzzing in GitPython. -The details about the latest OSS-Fuzz test status, including build logs and coverage reports, is made available +The latest details regarding OSS-Fuzz test status, including build logs and coverage reports, is made available at [this link](https://introspector.oss-fuzz.com/project-profile?project=gitpython). ## How to Contribute @@ -23,7 +23,7 @@ Areas that are particularly appreciated include: update of any size, please consider suggesting it! For everything else, such as expanding test coverage, optimizing test performance, or enhancing error detection -capabilities, jump in to the "Getting Started" section below. +capabilities, jump into the "Getting Started" section below. ## Getting Started with Fuzzing GitPython @@ -63,7 +63,7 @@ The `fuzzing/` directory is organized into three key areas: ### Fuzz Targets (`fuzz-targets/`) -Contains Python files for each fuzz test, targeting specific functionalities of GitPython. +Contains Python files for each fuzz test. **Things to Know**: @@ -81,7 +81,7 @@ Contains Python files for each fuzz test, targeting specific functionalities of Provides hints to the fuzzing engine about inputs that might trigger unique code paths. Each fuzz target may have a corresponding `.dict` file. For details on how these are used, refer -to [LibFuzzer documentation](https://llvm.org/docs/LibFuzzer.html#dictionaries). +to the [LibFuzzer documentation on the subject](https://llvm.org/docs/LibFuzzer.html#dictionaries). **Things to Know**: @@ -105,6 +105,11 @@ Includes scripts for building and integrating fuzz targets with OSS-Fuzz: - **`build.sh`** - Executed within the Docker container, this script builds fuzz targets with necessary instrumentation and prepares seed corpora and dictionaries for use. +**Where to learn more:** + +- [OSS-Fuzz documentation on the build.sh](https://google.github.io/oss-fuzz/getting-started/new-project-guide/#buildsh) +- [See GitPython's build.sh and Dockerfile in the OSS-Fuzz repository](https://github.com/google/oss-fuzz/tree/master/projects/gitpython) + ## Running Fuzzers Locally ### Direct Execution of Fuzz Targets @@ -153,9 +158,21 @@ python infra/helper.py check_build gitpython Execute the desired fuzz target: ```shell -python infra/helper.py run_fuzzer gitpython $FUZZ_TARGET +python infra/helper.py run_fuzzer gitpython $FUZZ_TARGET -- -max_total_time=60 -print_final_stats=1 ``` +> [!TIP] +> In the example above, the "`-- -max_total_time=60 -print_final_stats=1`" portion of the command is optional but quite +> useful. +> +> Every argument provided after "`--`" in the above command is passed to the fuzzing engine directly. In this case: +> - `-max_total_time=60` tells the LibFuzzer to stop execution after 60 seconds have elapsed. +> - `-print_final_stats=1` tells the LibFuzzer to print a summary of useful metrics about the target run upon + completion. +> +> But almost any [LibFuzzer option listed in the documentation](https://llvm.org/docs/LibFuzzer.html#options) should +> work as well. + #### Next Steps For detailed instructions on advanced features like reproducing OSS-Fuzz issues or using the Fuzz Introspector, refer From 576a858b7298bd14b1e87b118150df963af447dd Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 11 Apr 2024 22:06:05 -0400 Subject: [PATCH 0990/1392] Updates to support easily running OSS-Fuzz using local repo sources - Updates the fuzzing documentation to include steps for working with locally modified versions of the gitpython repository. - Updates the build.sh script to make the fuzz target search path more specific, reducing the risk of local OSS-Fuzz builds picking up files located outside of where we expect them (for example, in a .venv directory.) - add artifacts produced by local OSS-Fuzz runs to gitignore --- .gitignore | 3 +++ fuzzing/README.md | 19 +++++++++++++++++-- fuzzing/oss-fuzz-scripts/build.sh | 2 +- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 7765293d8..d85569405 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,6 @@ output.txt # Finder metadata .DS_Store + +# Files created by OSS-Fuzz when running locally +fuzz_*.pkg.spec diff --git a/fuzzing/README.md b/fuzzing/README.md index 97a62724a..acaf17d06 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -134,8 +134,10 @@ containers. Set environment variables to simplify command usage: ```shell -export SANITIZER=address # Can be either 'address' or 'undefined'. -export FUZZ_TARGET=fuzz_config # specify the fuzz target without the .py extension. +# $SANITIZER can be either 'address' or 'undefined': +export SANITIZER=address +# specify the fuzz target without the .py extension: +export FUZZ_TARGET=fuzz_config ``` #### Build and Run @@ -149,6 +151,19 @@ python infra/helper.py build_image gitpython python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython ``` +> [!TIP] +> The `build_fuzzers` command above accepts a local file path pointing to your gitpython repository clone as the last +> argument. +> This makes it easy to build fuzz targets you are developing locally in this repository without changing anything in +> the OSS-Fuzz repo! +> For example, if you have cloned this repository (or a fork of it) into: `~/code/GitPython` +> Then running this command would build new or modified fuzz targets using the `~/code/GitPython/fuzzing/fuzz-targets` +> directory: +> ```shell +> python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython ~/code/GitPython +> ``` + + Verify the build of your fuzzers with the optional `check_build` command: ```shell diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh index fdab7a1e0..aff1c4347 100644 --- a/fuzzing/oss-fuzz-scripts/build.sh +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -13,7 +13,7 @@ find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name -exec cp {} "$OUT" \; # Build fuzzers in $OUT. -find "$SRC" -name 'fuzz_*.py' -print0 | while IFS= read -r -d $'\0' fuzz_harness; do +find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d $'\0' fuzz_harness; do compile_python_fuzzer "$fuzz_harness" common_base_dictionary_filename="$SEED_DATA_DIR/__base.dict" From 5e56e96821878bd2808c640b8b39f84738ed8cf8 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 11 Apr 2024 23:37:54 -0400 Subject: [PATCH 0991/1392] Clarify documentation - Fix typos in the documentation on dictionaries - Link to the fuzzing directory in the main README where it is referenced. --- README.md | 2 +- fuzzing/README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 39f5496dc..b4cdc0a97 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,7 @@ Please have a look at the [contributions file][contributing]. [3-Clause BSD License](https://opensource.org/license/bsd-3-clause/), also known as the New BSD License. See the [LICENSE file][license]. > [!NOTE] -> There are two special case files located in the `fuzzzing/` directory that are licensed differently: +> There are two special case files located in the [`fuzzzing/` directory](./fuzzing) that are licensed differently: > > `fuzz_config.py` and `fuzz_tree.py` were migrated here from the OSS-Fuzz project repository where they were initially > created and retain the original licence and copyright notice (Apache License, Version 2.0 and Copyright 2023 Google diff --git a/fuzzing/README.md b/fuzzing/README.md index acaf17d06..c57e31d86 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -80,8 +80,8 @@ Contains Python files for each fuzz test. ### Dictionaries (`dictionaries/`) Provides hints to the fuzzing engine about inputs that might trigger unique code paths. Each fuzz target may have a -corresponding `.dict` file. For details on how these are used, refer -to the [LibFuzzer documentation on the subject](https://llvm.org/docs/LibFuzzer.html#dictionaries). +corresponding `.dict` file. For information about dictionary syntax, refer to +the [LibFuzzer documentation on the subject](https://llvm.org/docs/LibFuzzer.html#dictionaries). **Things to Know**: @@ -92,7 +92,7 @@ to the [LibFuzzer documentation on the subject](https://llvm.org/docs/LibFuzzer. existing file here. - Development or updates to dictionaries should reflect the varied formats and edge cases relevant to the functionalities under test. -- Example dictionaries (some of which are used to build the default dictionaries mentioned above) are can be found here: +- Example dictionaries (some of which are used to build the default dictionaries mentioned above) can be found here: - [AFL++ dictionary repository](https://github.com/AFLplusplus/AFLplusplus/tree/stable/dictionaries#readme) - [Google/fuzzing dictionary repository](https://github.com/google/fuzzing/tree/master/dictionaries) From 2041ba9972e7720f05bf570e2304fc0a5a2463d7 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Sun, 14 Apr 2024 22:59:01 -0400 Subject: [PATCH 0992/1392] Use gitpython-developers org ownd repository for seed corpra This repo was created after discussion in PR #1901. --- fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh index 43c21a8da..881161fae 100644 --- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -36,9 +36,9 @@ download_and_concatenate_common_dictionaries() { fetch_seed_corpra() { # Seed corpus zip files are hosted in a separate repository to avoid additional bloat in this repo. - git clone --depth 1 https://github.com/DaveLak/oss-fuzz-inputs.git oss-fuzz-inputs && - rsync -avc oss-fuzz-inputs/gitpython/corpra/ "$SEED_DATA_DIR/" && - rm -rf oss-fuzz-inputs; # Clean up the cloned repo to keep the Docker image as slim as possible. + git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git qa-assets && + rsync -avc qa-assets/gitpython/corpra/ "$SEED_DATA_DIR/" && + rm -rf qa-assets; # Clean up the cloned repo to keep the Docker image as slim as possible. } ######################## From 945a767ccd13c84946b2a49fbde4227fdfc84a26 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 02:06:50 -0400 Subject: [PATCH 0993/1392] Updates to comply with the terms of the Apache License Addresses feedback and encorperates suggestions from PR #1901 to ensure that the Apache License requirements are met for the two files that they apply to, and the documentation pertaining to licensing of the files in this repository is clear and concise. The contects of LICENSE-APACHE were coppied from the LICENSE file of the OSS-Fuzz repository that the two fuzz harnesses came from as of commit: https://github.com/google/oss-fuzz/blob/c2c0632831767ff05c568e7b552cef2801d739ff/LICENSE --- README.md | 13 +- fuzzing/LICENSE-APACHE | 201 ++++++++++++++++++++++++++++ fuzzing/README.md | 12 ++ fuzzing/fuzz-targets/fuzz_config.py | 6 + fuzzing/fuzz-targets/fuzz_tree.py | 6 + 5 files changed, 227 insertions(+), 11 deletions(-) create mode 100644 fuzzing/LICENSE-APACHE diff --git a/README.md b/README.md index b4cdc0a97..987e40e6c 100644 --- a/README.md +++ b/README.md @@ -240,17 +240,8 @@ Please have a look at the [contributions file][contributing]. [3-Clause BSD License](https://opensource.org/license/bsd-3-clause/), also known as the New BSD License. See the [LICENSE file][license]. -> [!NOTE] -> There are two special case files located in the [`fuzzzing/` directory](./fuzzing) that are licensed differently: -> -> `fuzz_config.py` and `fuzz_tree.py` were migrated here from the OSS-Fuzz project repository where they were initially -> created and retain the original licence and copyright notice (Apache License, Version 2.0 and Copyright 2023 Google -> LLC respectively.) -> -> - **These files do not impact the licence under which GitPython releases or source code are distributed.** -> - The files located in the `fuzzzing/` directory are part of the project test suite and neither packaged nor distributed as - part of any release. - +Two files exclusively used for fuzz testing are subject to [a separate license, detailed here](./fuzzing/README.md#license). +These files are not included in the wheel or sdist packages published by the maintainers of GitPython. [contributing]: https://github.com/gitpython-developers/GitPython/blob/main/CONTRIBUTING.md [license]: https://github.com/gitpython-developers/GitPython/blob/main/LICENSE diff --git a/fuzzing/LICENSE-APACHE b/fuzzing/LICENSE-APACHE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/fuzzing/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/fuzzing/README.md b/fuzzing/README.md index c57e31d86..09d6fc003 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -193,6 +193,18 @@ python infra/helper.py run_fuzzer gitpython $FUZZ_TARGET -- -max_total_time=60 - For detailed instructions on advanced features like reproducing OSS-Fuzz issues or using the Fuzz Introspector, refer to [the official OSS-Fuzz documentation][oss-fuzz-docs]. +## LICENSE + +All files located within the `fuzzing/` directory are subject to [the same license](../LICENSE) +as [the other files in this repository](../README.md#license) with two exceptions: + +Two files located in this directory, [`fuzz_config.py`](./fuzz-targets/fuzz_config.py) +and [`fuzz_tree.py`](./fuzz-targets/fuzz_tree.py), have been migrated here from the OSS-Fuzz project repository where +they were originally created. As such, these two files retain their original license and copyright notice (Apache +License, Version 2.0 and Copyright 2023 Google LLC respectively.) Each file includes a notice in their respective header +comments stating that they have been modified. [LICENSE-APACHE](./LICENSE-APACHE) contains the original license used by +the OSS-Fuzz project repository at the time they were migrated. + [oss-fuzz-repo]: https://github.com/google/oss-fuzz [oss-fuzz-docs]: https://google.github.io/oss-fuzz diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py index 1403c96e4..fc2f0960a 100644 --- a/fuzzing/fuzz-targets/fuzz_config.py +++ b/fuzzing/fuzz-targets/fuzz_config.py @@ -12,6 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +############################################################################### +# Note: This file has been modified by contributors to GitPython. +# The original state of this file may be referenced here: +# https://github.com/google/oss-fuzz/commit/f26f254558fc48f3c9bc130b10507386b94522da +############################################################################### import atheris import sys import io diff --git a/fuzzing/fuzz-targets/fuzz_tree.py b/fuzzing/fuzz-targets/fuzz_tree.py index 53258fb1e..b4e0e6b55 100644 --- a/fuzzing/fuzz-targets/fuzz_tree.py +++ b/fuzzing/fuzz-targets/fuzz_tree.py @@ -12,6 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +############################################################################### +# Note: This file has been modified by contributors to GitPython. +# The original state of this file may be referenced here: +# https://github.com/google/oss-fuzz/commit/f26f254558fc48f3c9bc130b10507386b94522da +############################################################################### import atheris import io import sys From 68194a913fa0d9f601a55fcd08ff13b7ac35be75 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 14:41:36 -0400 Subject: [PATCH 0994/1392] Remove shebangs from fuzz harnesses Prefer executing these files using the OSS-Fuzz or `python` command methods outlined in the `fuzzing/README`. Based on feedback and discussion on: https://github.com/gitpython-developers/GitPython/pull/1901 --- fuzzing/fuzz-targets/fuzz_config.py | 1 - fuzzing/fuzz-targets/fuzz_tree.py | 1 - 2 files changed, 2 deletions(-) diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py index fc2f0960a..0a06956c8 100644 --- a/fuzzing/fuzz-targets/fuzz_config.py +++ b/fuzzing/fuzz-targets/fuzz_config.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 # Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/fuzzing/fuzz-targets/fuzz_tree.py b/fuzzing/fuzz-targets/fuzz_tree.py index b4e0e6b55..464235098 100644 --- a/fuzzing/fuzz-targets/fuzz_tree.py +++ b/fuzzing/fuzz-targets/fuzz_tree.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 # Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); From 8954c7151e098a0b12d4d2dec277fe6c63980579 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 14:45:49 -0400 Subject: [PATCH 0995/1392] Replace shebang in `build.sh` with ShellCheck directive This script is meant to be sourced by the OSS-Fuzz file of the same name, rather than executed directly. The shebang may lead to the incorrect assumption that the script is meant for direct execution. Replacing it with this directive instructs ShellCheck to treat the script as a Bash script, regardless of how it is executed. Based @EliahKagan's suggestion and feedback on: https://github.com/gitpython-developers/GitPython/pull/1901 --- fuzzing/oss-fuzz-scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh index aff1c4347..4c7de8799 100644 --- a/fuzzing/oss-fuzz-scripts/build.sh +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +# shellcheck shell=bash set -euo pipefail From b0a5b8e66c4da3d603d8e27a71c70aaad53542b8 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 14:59:05 -0400 Subject: [PATCH 0996/1392] Set executable bit on `container-environment-bootstrap.sh` This script is executed directly, not sourced as is the case with `build.sh`, so it should have an executable bit set to avoid ambiguity. Based @EliahKagan's suggestion and feedback on: https://github.com/gitpython-developers/GitPython/pull/1901 --- fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh old mode 100644 new mode 100755 From 25f360090cb6a7fd0f01bc127f2a2280659757a2 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 15:31:05 -0400 Subject: [PATCH 0997/1392] Minor clarity improvements in `fuzzing/README.md` - Make the link text for the OSS-Fuzz test status URL more descriptive - Fix capitalization of GitPython repository name Based @EliahKagan's suggestion and feedback on: https://github.com/gitpython-developers/GitPython/pull/1901 --- fuzzing/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fuzzing/README.md b/fuzzing/README.md index 09d6fc003..65e311d4a 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -6,8 +6,8 @@ This directory contains files related to GitPython's suite of fuzz tests that ar infrastructure provided by [OSS-Fuzz][oss-fuzz-repo]. This document aims to provide necessary information for working with fuzzing in GitPython. -The latest details regarding OSS-Fuzz test status, including build logs and coverage reports, is made available -at [this link](https://introspector.oss-fuzz.com/project-profile?project=gitpython). +The latest details regarding OSS-Fuzz test status, including build logs and coverage reports, is available +on [the Open Source Fuzzing Introspection website](https://introspector.oss-fuzz.com/project-profile?project=gitpython). ## How to Contribute @@ -152,7 +152,7 @@ python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython ``` > [!TIP] -> The `build_fuzzers` command above accepts a local file path pointing to your gitpython repository clone as the last +> The `build_fuzzers` command above accepts a local file path pointing to your GitPython repository clone as the last > argument. > This makes it easy to build fuzz targets you are developing locally in this repository without changing anything in > the OSS-Fuzz repo! From d79c176384f1a5b6cb615f500037dbcecd9ee7d9 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 16:10:08 -0400 Subject: [PATCH 0998/1392] Simplify read delimiter to use empty string in fuzz harness loop Replaces the null character delimiter `-d $'\0'` with the simpler empty string `-d ''` in the fuzzing harness build loop. This changes leverages the Bash `read` builtin behavior to avoid unnecessary complexity and improving script readability. Based @EliahKagan's suggestion and feedback on: https://github.com/gitpython-developers/GitPython/pull/1901 --- fuzzing/oss-fuzz-scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh index 4c7de8799..a412a1d15 100644 --- a/fuzzing/oss-fuzz-scripts/build.sh +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -13,7 +13,7 @@ find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name -exec cp {} "$OUT" \; # Build fuzzers in $OUT. -find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d $'\0' fuzz_harness; do +find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do compile_python_fuzzer "$fuzz_harness" common_base_dictionary_filename="$SEED_DATA_DIR/__base.dict" From e038526b846f4bc5e75a91c736f3384616800aa1 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 16:27:43 -0400 Subject: [PATCH 0999/1392] Remove unnecessary semicolon for consistent script formatting Based @EliahKagan's suggestion and feedback on: https://github.com/gitpython-developers/GitPython/pull/1901 --- .../container-environment-bootstrap.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh index 881161fae..87f817993 100755 --- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -38,19 +38,19 @@ fetch_seed_corpra() { # Seed corpus zip files are hosted in a separate repository to avoid additional bloat in this repo. git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git qa-assets && rsync -avc qa-assets/gitpython/corpra/ "$SEED_DATA_DIR/" && - rm -rf qa-assets; # Clean up the cloned repo to keep the Docker image as slim as possible. + rm -rf qa-assets # Clean up the cloned repo to keep the Docker image as slim as possible. } ######################## # Main execution logic # ######################## -fetch_seed_corpra; +fetch_seed_corpra download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \ "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \ - "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict"; + "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict" # The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below. -python3 -m pip install --upgrade pip; -python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0'; # Uses the latest versions know to work at the time of this commit. +python3 -m pip install --upgrade pip +python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0' # Uses the latest versions know to work at the time of this commit. From d25ae2def1f995afcb7fad69250b12f5bf07b3bb Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 16:38:21 -0400 Subject: [PATCH 1000/1392] Fix various misspellings of "corpora" & improve script comments A misspelling in the https://github.com/gitpython-developers/qa-assets repository is still present here. It will need to be fixed in that repository first. "corpora" is a difficult word to spell consistently I guess. This made for a good opportunity to improve the phrasing of two other comments at at least. Based @EliahKagan's suggestion and feedback on: https://github.com/gitpython-developers/GitPython/pull/1901 --- fuzzing/oss-fuzz-scripts/build.sh | 4 ++-- .../oss-fuzz-scripts/container-environment-bootstrap.sh | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh index a412a1d15..ab46ec7a2 100644 --- a/fuzzing/oss-fuzz-scripts/build.sh +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -4,7 +4,7 @@ set -euo pipefail python3 -m pip install . -# Directory to look in for dictionaries, options files, and seed corpa: +# Directory to look in for dictionaries, options files, and seed corpora: SEED_DATA_DIR="$SRC/seed_data" find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name '*.dict' \) \ @@ -27,7 +27,7 @@ find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d # If a dictionary file for this fuzzer already exists and is not empty, # we append a new line to the end of it before appending any new entries. # - # libfuzzer will happily ignore multiple empty lines in a dictionary but crash + # LibFuzzer will happily ignore multiple empty lines in a dictionary but fail with an error # if any single line has incorrect syntax (e.g., if we accidentally add two entries to the same line.) # See docs for valid syntax: https://llvm.org/docs/LibFuzzer.html#id32 echo >>"$output_file" diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh index 87f817993..0be012ccd 100755 --- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -34,7 +34,7 @@ download_and_concatenate_common_dictionaries() { done } -fetch_seed_corpra() { +fetch_seed_corpora() { # Seed corpus zip files are hosted in a separate repository to avoid additional bloat in this repo. git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git qa-assets && rsync -avc qa-assets/gitpython/corpra/ "$SEED_DATA_DIR/" && @@ -45,7 +45,7 @@ fetch_seed_corpra() { # Main execution logic # ######################## -fetch_seed_corpra +fetch_seed_corpora download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \ "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \ @@ -53,4 +53,5 @@ download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \ # The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below. python3 -m pip install --upgrade pip -python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0' # Uses the latest versions know to work at the time of this commit. + # Upgrade to the latest versions known to work at the time the below changes were introduced: +python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0' From 23a505f3ef51c4c26998fed924f4edad2438c757 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Wed, 17 Apr 2024 19:40:44 -0400 Subject: [PATCH 1001/1392] Remove comment suggesting the `undefined` sanitizer is a valid option Also makes come structural improvements to how the local instructions for running OSS-Fuzz are presented now that only the single `address` sanitizer is a valid option. The `undefined` sanitizer was removed from GitPython's `project.yaml` OSS-Fuzz configuration file at the request of OSS-Fuzz project reviewers in https://github.com/google/oss-fuzz/pull/11803. The `undefined` sanitizer is only useful in Python projects that use native exstensions (such as C, C++, Rust, ect.), which GitPython does not currently do. This commit updates the `fuzzing/README` reference to that sanitizer accoirdingly. See: - https://github.com/google/oss-fuzz/pull/11803/commits/b210fb21427f1f994c91f07e95ca0cc977f61f66 - https://github.com/google/oss-fuzz/pull/11803#discussion_r1569160945 --- fuzzing/README.md | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/fuzzing/README.md b/fuzzing/README.md index 65e311d4a..ab9f6a63f 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -129,18 +129,7 @@ This approach uses Docker images provided by OSS-Fuzz for building and running f comprehensive features but requires a local clone of the OSS-Fuzz repository and sufficient disk space for Docker containers. -#### Preparation - -Set environment variables to simplify command usage: - -```shell -# $SANITIZER can be either 'address' or 'undefined': -export SANITIZER=address -# specify the fuzz target without the .py extension: -export FUZZ_TARGET=fuzz_config -``` - -#### Build and Run +#### Build the Execution Environment Clone the OSS-Fuzz repository and prepare the Docker environment: @@ -148,7 +137,7 @@ Clone the OSS-Fuzz repository and prepare the Docker environment: git clone --depth 1 https://github.com/google/oss-fuzz.git oss-fuzz cd oss-fuzz python infra/helper.py build_image gitpython -python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython +python infra/helper.py build_fuzzers --sanitizer address gitpython ``` > [!TIP] @@ -160,16 +149,25 @@ python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython > Then running this command would build new or modified fuzz targets using the `~/code/GitPython/fuzzing/fuzz-targets` > directory: > ```shell -> python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython ~/code/GitPython +> python infra/helper.py build_fuzzers --sanitizer address gitpython ~/code/GitPython > ``` - Verify the build of your fuzzers with the optional `check_build` command: ```shell python infra/helper.py check_build gitpython ``` +#### Run a Fuzz Target + +Setting an environment variable for the fuzz target argument of the execution command makes it easier to quickly select +a different target between runs: + +```shell +# specify the fuzz target without the .py extension: +export FUZZ_TARGET=fuzz_config +``` + Execute the desired fuzz target: ```shell From 1d54d4b0b2bdba60cc742f73a4b8d5a88cce8f64 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Wed, 17 Apr 2024 22:11:00 -0400 Subject: [PATCH 1002/1392] Remove unintentional leading space from comment --- fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh index 0be012ccd..662808e27 100755 --- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -53,5 +53,5 @@ download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \ # The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below. python3 -m pip install --upgrade pip - # Upgrade to the latest versions known to work at the time the below changes were introduced: +# Upgrade to the latest versions known to work at the time the below changes were introduced: python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0' From fdce8375c80abd02b1dc08bd218f09849fea8233 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Sat, 20 Apr 2024 16:48:00 -0400 Subject: [PATCH 1003/1392] Dockerized "Direct Execution of Fuzz Targets" Adds a Dockerfile to enable easily executing the fuzz targets directly inside a container environment instead of directly on a host machine. This addresses concerns raised in PR #1901 related to how `fuzz_tree.py` writes to the real `/tmp` directory of the file system it is executed on as part of setting up its own test fixtures, but also makes for an easier to use development workflow. See this related comment on PR #1901 for additional context: https://github.com/gitpython-developers/GitPython/pull/1901#issuecomment-2063818998 --- fuzzing/README.md | 43 ++++++++++++++++++++++++---- fuzzing/local-dev-helpers/Dockerfile | 22 ++++++++++++++ 2 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 fuzzing/local-dev-helpers/Dockerfile diff --git a/fuzzing/README.md b/fuzzing/README.md index ab9f6a63f..0a62b4c85 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -44,8 +44,7 @@ capabilities, jump into the "Getting Started" section below. ### Setting Up Your Local Environment Before contributing to fuzzing efforts, ensure Python and Docker are installed on your machine. Docker is required for -running fuzzers in containers provided by OSS-Fuzz. [Install Docker](https://docs.docker.com/get-docker/) following the -official guide if you do not already have it. +running fuzzers in containers provided by OSS-Fuzz and for safely executing test files directly. [Install Docker](https://docs.docker.com/get-docker/) following the official guide if you do not already have it. ### Understanding Existing Fuzz Targets @@ -110,19 +109,51 @@ Includes scripts for building and integrating fuzz targets with OSS-Fuzz: - [OSS-Fuzz documentation on the build.sh](https://google.github.io/oss-fuzz/getting-started/new-project-guide/#buildsh) - [See GitPython's build.sh and Dockerfile in the OSS-Fuzz repository](https://github.com/google/oss-fuzz/tree/master/projects/gitpython) +### Local Development Helpers (`local-dev-helpers/`) + +Contains tools to make local development tasks easier. +See [the "Running Fuzzers Locally" section below](#running-fuzzers-locally) for further documentation and use cases related to files found here. + ## Running Fuzzers Locally +> [!WARNING] +> **Some fuzz targets in this repository write to the filesystem** during execution. +> For that reason, it is strongly recommended to **always use Docker when executing fuzz targets**, even when it may be +> possible to do so without it. +> +> Although [I/O operations such as writing to disk are not considered best practice](https://github.com/google/fuzzing/blob/master/docs/good-fuzz-target.md#io), the current implementation of at least one test requires it. +> See [the "Setting Up Your Local Environment" section above](#setting-up-your-local-environment) if you do not already have Docker installed on your machine. +> +> PRs that replace disk I/O with in-memory alternatives are very much welcomed! + ### Direct Execution of Fuzz Targets -For quick testing of changes, [Atheris][atheris-repo] makes it possible to execute a fuzz target directly: +Directly executing fuzz targets allows for quick iteration and testing of changes which can be helpful during early +development of new fuzz targets or for validating changes made to an existing test. +The [Dockerfile](./local-dev-helpers/Dockerfile) located in the `local-dev-helpers/` subdirectory provides a lightweight +container environment preconfigured with [Atheris][atheris-repo] that makes it easy to execute a fuzz target directly. + +**From the root directory of your GitPython repository clone**: -1. Install Atheris following the [installation guide][atheris-repo] for your operating system. -2. Execute a fuzz target, for example: +1. Build the local development helper image: ```shell -python fuzzing/fuzz-targets/fuzz_config.py +docker build -f fuzzing/local-dev-helpers/Dockerfile -t gitpython-fuzzdev . ``` +2. Then execute a fuzz target inside the image, for example: + +```shell + docker run -it -v "$PWD":/src gitpython-fuzzdev python fuzzing/fuzz-targets/fuzz_config.py -atheris_runs=10000 +``` + +The above command executes [`fuzz_config.py`](./fuzz-targets/fuzz_config.py) and exits after `10000` runs, or earlier if +the fuzzer finds an error. + +Docker CLI's `-v` flag specifies a volume mount in Docker that maps the directory in which the command is run (which +should be the root directory of your local GitPython clone) to a directory inside the container, so any modifications +made between invocations will be reflected immediately without the need to rebuild the image each time. + ### Running OSS-Fuzz Locally This approach uses Docker images provided by OSS-Fuzz for building and running fuzz tests locally. It offers diff --git a/fuzzing/local-dev-helpers/Dockerfile b/fuzzing/local-dev-helpers/Dockerfile new file mode 100644 index 000000000..77808ed1d --- /dev/null +++ b/fuzzing/local-dev-helpers/Dockerfile @@ -0,0 +1,22 @@ +# syntax=docker/dockerfile:1 + +# Use the same Python version as OSS-Fuzz to accidental incompatibilities in test code +FROM python:3.8-slim + +LABEL project="GitPython Fuzzing Local Dev Helper" + +WORKDIR /src + +COPY . . + +# Update package managers, install necessary packages, and cleanup unnecessary files in a single RUN to keep the image smaller. +RUN apt-get update && \ + apt-get install --no-install-recommends -y git && \ + python -m pip install --upgrade pip && \ + python -m pip install atheris && \ + python -m pip install -e . && \ + apt-get clean && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* /root/.cache + +CMD ["bash"] From f1451219c5a3a221615d3d38ac251bf5bbe46119 Mon Sep 17 00:00:00 2001 From: DaveLak Date: Sun, 21 Apr 2024 12:19:57 -0400 Subject: [PATCH 1004/1392] Fix Atheris install in local dev helper Docker image The Atheris package bundles a binary that supplies libFuzzer on some host machines, but in some cases (such as ARM based mac hosts) Atheris seems to require building libFuzzer at install time while pip builds the wheel. In the latter case, clang and related dependencies must be present and available for the build, which itself requires using a non "slim" version of the Python base image and not passing the `--no-install-recommends` flag to `apt-get install` as both prevent the required related libraries from being automatically installed. It is also worth noting that at the time of this commit, the default version of LLVM & Clang installed when `clang` is installed from `apt` is version 14, while the latest stable version is 17 and OSS-Fuzz uses 15. The decision to install the default version (14) available via the debian repos was intentional because a) it appears to work fine for our needs and Atheris version b) specifying a different version requires more complexity depending on install method, but the goal of this Dockerfile is simplicity and low maintenance. If it becomes neccissary to upgrade Clang/LLVM in the future, one option to consider besides installing from source is the apt repository maintained by the LLVM project: https://apt.llvm.org/ See the discussion in this issue for additional context to this change: https://github.com/gitpython-developers/GitPython/pull/1904 --- fuzzing/local-dev-helpers/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fuzzing/local-dev-helpers/Dockerfile b/fuzzing/local-dev-helpers/Dockerfile index 77808ed1d..426de05dd 100644 --- a/fuzzing/local-dev-helpers/Dockerfile +++ b/fuzzing/local-dev-helpers/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 # Use the same Python version as OSS-Fuzz to accidental incompatibilities in test code -FROM python:3.8-slim +FROM python:3.8-bookworm LABEL project="GitPython Fuzzing Local Dev Helper" @@ -11,12 +11,12 @@ COPY . . # Update package managers, install necessary packages, and cleanup unnecessary files in a single RUN to keep the image smaller. RUN apt-get update && \ - apt-get install --no-install-recommends -y git && \ + apt-get install -y git clang && \ python -m pip install --upgrade pip && \ python -m pip install atheris && \ python -m pip install -e . && \ apt-get clean && \ apt-get autoremove -y && \ - rm -rf /var/lib/apt/lists/* /root/.cache + rm -rf /var/lib/apt/lists/* CMD ["bash"] From f4b95cf089706a29396b744b53a4ecdcc924d31c Mon Sep 17 00:00:00 2001 From: David Lakin Date: Mon, 22 Apr 2024 16:07:54 -0400 Subject: [PATCH 1005/1392] Fix Fuzzer Crash in ClusterFuzz Due to Missing Git Executable A Git executable is not globally available in the ClusterFuzz container environment where OSS-Fuzz executes fuzz tests, causing an error in the fuzz harnesses when GitPython attempts to initialize, crashing the tests before they can run. To avoid this issue, we bundle the `git` binary that is available in the OSS-Fuzz build container with the fuzz harness via Pyinstaller's `--add-binary` flag in `build.sh` and use GitPython's `git.refresh()` method inside a Pyinstaller runtime check to initialize GitPython with the bundled Git executable when running from the bundled application. In all other execution environments, we assume a `git` executable is available globally. Fixes: - https://github.com/gitpython-developers/GitPython/issues/1905 - https://github.com/google/oss-fuzz/issues/10600 --- fuzzing/fuzz-targets/fuzz_config.py | 9 +++++++-- fuzzing/fuzz-targets/fuzz_tree.py | 11 +++++++---- fuzzing/oss-fuzz-scripts/build.sh | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py index 0a06956c8..7623ab98f 100644 --- a/fuzzing/fuzz-targets/fuzz_config.py +++ b/fuzzing/fuzz-targets/fuzz_config.py @@ -20,16 +20,21 @@ import atheris import sys import io +import os from configparser import MissingSectionHeaderError, ParsingError with atheris.instrument_imports(): - from git import GitConfigParser + import git def TestOneInput(data): + if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + git.refresh(path_to_bundled_git_binary) + sio = io.BytesIO(data) sio.name = "/tmp/fuzzconfig.config" - git_config = GitConfigParser(sio) + git_config = git.GitConfigParser(sio) try: git_config.read() except (MissingSectionHeaderError, ParsingError, UnicodeDecodeError): diff --git a/fuzzing/fuzz-targets/fuzz_tree.py b/fuzzing/fuzz-targets/fuzz_tree.py index 464235098..7187c4a6f 100644 --- a/fuzzing/fuzz-targets/fuzz_tree.py +++ b/fuzzing/fuzz-targets/fuzz_tree.py @@ -24,11 +24,14 @@ import shutil with atheris.instrument_imports(): - from git.objects import Tree - from git.repo import Repo + import git def TestOneInput(data): + if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + git.refresh(path_to_bundled_git_binary) + fdp = atheris.FuzzedDataProvider(data) git_dir = "/tmp/.git" head_file = os.path.join(git_dir, "HEAD") @@ -46,9 +49,9 @@ def TestOneInput(data): os.mkdir(common_dir) os.mkdir(objects_dir) - _repo = Repo("/tmp/") + _repo = git.Repo("/tmp/") - fuzz_tree = Tree(_repo, Tree.NULL_BIN_SHA, 0, "") + fuzz_tree = git.Tree(_repo, git.Tree.NULL_BIN_SHA, 0, "") try: fuzz_tree._deserialize(io.BytesIO(data)) except IndexError: diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh index ab46ec7a2..be31ac32a 100644 --- a/fuzzing/oss-fuzz-scripts/build.sh +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -14,7 +14,7 @@ find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name # Build fuzzers in $OUT. find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do - compile_python_fuzzer "$fuzz_harness" + compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):." common_base_dictionary_filename="$SEED_DATA_DIR/__base.dict" if [[ -r "$common_base_dictionary_filename" ]]; then From 2b0a9693ea98ab7ff025a8ab1235b6f8ea0da676 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Mon, 22 Apr 2024 16:36:17 -0400 Subject: [PATCH 1006/1392] Add GitPython's standard license header comments to oss-fuzz scripts These files are already BSD-3-Clause even without the headers, but adding these comments and the `LICENSE-BSD` symlink to the root level `LICENSE` file are helpful to reinforce that there are only two particular files in the `fuzzing/` that are not under BSD-3-Clause. See: https://github.com/gitpython-developers/GitPython/pull/1901#discussion_r1567849271 --- fuzzing/LICENSE-BSD | 1 + fuzzing/oss-fuzz-scripts/build.sh | 3 +++ fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh | 4 ++++ 3 files changed, 8 insertions(+) create mode 120000 fuzzing/LICENSE-BSD diff --git a/fuzzing/LICENSE-BSD b/fuzzing/LICENSE-BSD new file mode 120000 index 000000000..ea5b60640 --- /dev/null +++ b/fuzzing/LICENSE-BSD @@ -0,0 +1 @@ +../LICENSE \ No newline at end of file diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh index ab46ec7a2..a79cbe895 100644 --- a/fuzzing/oss-fuzz-scripts/build.sh +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -1,4 +1,7 @@ # shellcheck shell=bash +# +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ set -euo pipefail diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh index 662808e27..76ec97c7f 100755 --- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +# +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + set -euo pipefail ################# From b021a76354bb2779fc8f43c647a3a85b67f3d01a Mon Sep 17 00:00:00 2001 From: David Lakin Date: Mon, 22 Apr 2024 16:43:03 -0400 Subject: [PATCH 1007/1392] Add GitPython's standard license header comments to top level scripts While discussing adding similar license comments to the shell scripts introduced in PR #1901, it was noticed that the shell scripts in the repository root directory did not have such comments and suggested that we could add them when the scripts in the `fuzzing/` directory were updated, so this commit does just that. See: https://github.com/gitpython-developers/GitPython/pull/1901#discussion_r1567849271 --- build-release.sh | 3 +++ check-version.sh | 3 +++ init-tests-after-clone.sh | 3 +++ 3 files changed, 9 insertions(+) diff --git a/build-release.sh b/build-release.sh index 49c13b93a..1a8dce2c2 100755 --- a/build-release.sh +++ b/build-release.sh @@ -1,5 +1,8 @@ #!/bin/bash # +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +# # This script builds a release. If run in a venv, it auto-installs its tools. # You may want to run "make release" instead of running this script directly. diff --git a/check-version.sh b/check-version.sh index dac386e46..579cf789f 100755 --- a/check-version.sh +++ b/check-version.sh @@ -1,5 +1,8 @@ #!/bin/bash # +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +# # This script checks if we are in a consistent state to build a new release. # See the release instructions in README.md for the steps to make this pass. # You may want to run "make release" instead of running this script directly. diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index 118e1de22..bfada01b0 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -1,4 +1,7 @@ #!/bin/sh +# +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ set -eu From 47e5738074e7f9acfb64d164206770bbd41685a0 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Mon, 22 Apr 2024 18:48:26 -0400 Subject: [PATCH 1008/1392] Fix IndexError in GitConfigParser when config value ends in new line Improve the guarding `if` check in `GitConfigParser`'s `string_decode` function to safely handle empty strings and prevent `IndexError`s when accessing string elements. This resolves an IndexError in the `GitConfigParser`'s `.read()` method when the config file contains a quoted value containing a trailing new line. Fixes: https://github.com/gitpython-developers/GitPython/issues/1887 --- fuzzing/fuzz-targets/fuzz_config.py | 8 ++------ git/config.py | 2 +- test/test_config.py | 8 ++++++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py index 0a06956c8..81dcf9a88 100644 --- a/fuzzing/fuzz-targets/fuzz_config.py +++ b/fuzzing/fuzz-targets/fuzz_config.py @@ -34,12 +34,8 @@ def TestOneInput(data): git_config.read() except (MissingSectionHeaderError, ParsingError, UnicodeDecodeError): return -1 # Reject inputs raising expected exceptions - except (IndexError, ValueError) as e: - if isinstance(e, IndexError) and "string index out of range" in str(e): - # Known possibility that might be patched - # See: https://github.com/gitpython-developers/GitPython/issues/1887 - pass - elif isinstance(e, ValueError) and "embedded null byte" in str(e): + except ValueError as e: + if isinstance(e, ValueError) and "embedded null byte" in str(e): # The `os.path.expanduser` function, which does not accept strings # containing null bytes might raise this. return -1 diff --git a/git/config.py b/git/config.py index 3ce9b123f..c9b49684c 100644 --- a/git/config.py +++ b/git/config.py @@ -452,7 +452,7 @@ def _read(self, fp: Union[BufferedReader, IO[bytes]], fpname: str) -> None: e = None # None, or an exception. def string_decode(v: str) -> str: - if v[-1] == "\\": + if v and v[-1] == "\\": v = v[:-1] # END cut trailing escapes to prevent decode error diff --git a/test/test_config.py b/test/test_config.py index 0911d0262..92997422d 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -142,6 +142,14 @@ def test_multi_line_config(self): ) self.assertEqual(len(config.sections()), 23) + def test_config_value_with_trailing_new_line(self): + config_content = b'[section-header]\nkey:"value\n"' + config_file = io.BytesIO(config_content) + config_file.name = "multiline_value.config" + + git_config = GitConfigParser(config_file) + git_config.read() # This should not throw an exception + def test_base(self): path_repo = fixture_path("git_config") path_global = fixture_path("git_config_global") From c2283f6e3566606300f64c44a12197f0b65f0d71 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 23 Apr 2024 08:21:58 +0200 Subject: [PATCH 1009/1392] Avoid unnecessary isinstance check --- fuzzing/fuzz-targets/fuzz_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py index 81dcf9a88..80ab7b08d 100644 --- a/fuzzing/fuzz-targets/fuzz_config.py +++ b/fuzzing/fuzz-targets/fuzz_config.py @@ -35,7 +35,7 @@ def TestOneInput(data): except (MissingSectionHeaderError, ParsingError, UnicodeDecodeError): return -1 # Reject inputs raising expected exceptions except ValueError as e: - if isinstance(e, ValueError) and "embedded null byte" in str(e): + if "embedded null byte" in str(e): # The `os.path.expanduser` function, which does not accept strings # containing null bytes might raise this. return -1 From 1a0ab5bbdaa9df5d04d1b6946af419492b650fce Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 26 Apr 2024 07:15:13 +0200 Subject: [PATCH 1010/1392] Use endswith() for more clarity --- git/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/config.py b/git/config.py index c9b49684c..de3508360 100644 --- a/git/config.py +++ b/git/config.py @@ -452,7 +452,7 @@ def _read(self, fp: Union[BufferedReader, IO[bytes]], fpname: str) -> None: e = None # None, or an exception. def string_decode(v: str) -> str: - if v and v[-1] == "\\": + if v and v.endswith("\\"): v = v[:-1] # END cut trailing escapes to prevent decode error From dac3535d3dc4aaff9bd98a6ea70f46b132537694 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Fri, 26 Apr 2024 18:13:11 -0400 Subject: [PATCH 1011/1392] Attempt 2 - Fix Missing Git Executable Causing ClusterFuzz Crash This is a second attempt at #1906 and should resolve: - https://github.com/gitpython-developers/GitPython/issues/1905 - https://github.com/google/oss-fuzz/issues/10600 PR #1906 had the right idea but wrong implementation, and the differences between the ClusterFuzz image that it was supposed to fix and the OSS-Fuzz image where the fix was tested led to the issue not being fully resolved. The root cause of the issue is the same: A Git executable is not globally available in the ClusterFuzz container environment where OSS-Fuzz executes fuzz tests. #1906 attempted to fix the issue by bundling the Git binary and using GitPython's `git.refresh()` method to set it inside the `TestOneInput` function of the test harness. However, GitPython attempts to set the binary at import time via its `__init__` hook, and crashes the test if no executable is found during the import. This issue is fixed here by setting the environment variable that GitPython looks in before importing it, so it's available for the import. This was tested by setting the `$PATH` to an empty string inside the test files, which reproduced the crash, then adding the changes introduced here with `$PATH` still empty, which avoided the crash indicating that the bundled Git executable is working as expected. --- fuzzing/fuzz-targets/fuzz_config.py | 8 ++++---- fuzzing/fuzz-targets/fuzz_tree.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py index 6f2caad4b..4eddc32ff 100644 --- a/fuzzing/fuzz-targets/fuzz_config.py +++ b/fuzzing/fuzz-targets/fuzz_config.py @@ -23,15 +23,15 @@ import os from configparser import MissingSectionHeaderError, ParsingError +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + with atheris.instrument_imports(): import git def TestOneInput(data): - if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): - path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) - git.refresh(path_to_bundled_git_binary) - sio = io.BytesIO(data) sio.name = "/tmp/fuzzconfig.config" git_config = git.GitConfigParser(sio) diff --git a/fuzzing/fuzz-targets/fuzz_tree.py b/fuzzing/fuzz-targets/fuzz_tree.py index 7187c4a6f..4e2038add 100644 --- a/fuzzing/fuzz-targets/fuzz_tree.py +++ b/fuzzing/fuzz-targets/fuzz_tree.py @@ -23,15 +23,15 @@ import os import shutil +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + with atheris.instrument_imports(): import git def TestOneInput(data): - if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): - path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) - git.refresh(path_to_bundled_git_binary) - fdp = atheris.FuzzedDataProvider(data) git_dir = "/tmp/.git" head_file = os.path.join(git_dir, "HEAD") From c84e643c6aa177f364ebe28e4c7bab1e37fb0242 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Sun, 28 Apr 2024 22:41:11 -0400 Subject: [PATCH 1012/1392] Replace the suboptimal fuzz_tree harness with a better alternative As discussed in the initial fuzzing integration PR[^1], `fuzz_tree.py`'s implementation was not ideal in terms of coverage and its reading/writing to hard-coded paths inside `/tmp` was problematic as (among other concerns), it causes intermittent crashes on ClusterFuzz[^2] when multiple workers execute the test at the same time on the same machine. The changes here replace `fuzz_tree.py` completely with a completely new `fuzz_repo.py` fuzz target which: - Uses `tempfile.TemporaryDirectory()` to safely manage tmpdir creation and tear down, including during multi-worker execution runs. - Retains the same feature coverage as `fuzz_tree.py`, but it also adds considerably more from much smaller data inputs and with less memory consumed (and it doesn't even have a seed corpus or target specific dictionary yet.) - Can likely be improved further in the future by exercising additional features of `Repo` to the harness. Because `fuzz_tree.py` was removed and `fuzz_repo.py` was not derived from it, the Apache License call outs in the docs were also updated as they only apply to the singe `fuzz_config.py` file now. [^1]: https://github.com/gitpython-developers/GitPython/pull/1901#discussion_r1565001609 [^2]: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=68355 --- README.md | 4 +- fuzzing/README.md | 16 +++---- fuzzing/dictionaries/fuzz_tree.dict | 13 ------ fuzzing/fuzz-targets/fuzz_repo.py | 47 ++++++++++++++++++++ fuzzing/fuzz-targets/fuzz_tree.py | 67 ----------------------------- 5 files changed, 57 insertions(+), 90 deletions(-) delete mode 100644 fuzzing/dictionaries/fuzz_tree.dict create mode 100644 fuzzing/fuzz-targets/fuzz_repo.py delete mode 100644 fuzzing/fuzz-targets/fuzz_tree.py diff --git a/README.md b/README.md index 987e40e6c..d365a6584 100644 --- a/README.md +++ b/README.md @@ -240,8 +240,8 @@ Please have a look at the [contributions file][contributing]. [3-Clause BSD License](https://opensource.org/license/bsd-3-clause/), also known as the New BSD License. See the [LICENSE file][license]. -Two files exclusively used for fuzz testing are subject to [a separate license, detailed here](./fuzzing/README.md#license). -These files are not included in the wheel or sdist packages published by the maintainers of GitPython. +One file exclusively used for fuzz testing is subject to [a separate license, detailed here](./fuzzing/README.md#license). +This file is not included in the wheel or sdist packages published by the maintainers of GitPython. [contributing]: https://github.com/gitpython-developers/GitPython/blob/main/CONTRIBUTING.md [license]: https://github.com/gitpython-developers/GitPython/blob/main/LICENSE diff --git a/fuzzing/README.md b/fuzzing/README.md index 0a62b4c85..9d02bf72f 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -225,14 +225,14 @@ to [the official OSS-Fuzz documentation][oss-fuzz-docs]. ## LICENSE All files located within the `fuzzing/` directory are subject to [the same license](../LICENSE) -as [the other files in this repository](../README.md#license) with two exceptions: - -Two files located in this directory, [`fuzz_config.py`](./fuzz-targets/fuzz_config.py) -and [`fuzz_tree.py`](./fuzz-targets/fuzz_tree.py), have been migrated here from the OSS-Fuzz project repository where -they were originally created. As such, these two files retain their original license and copyright notice (Apache -License, Version 2.0 and Copyright 2023 Google LLC respectively.) Each file includes a notice in their respective header -comments stating that they have been modified. [LICENSE-APACHE](./LICENSE-APACHE) contains the original license used by -the OSS-Fuzz project repository at the time they were migrated. +as [the other files in this repository](../README.md#license) with one exception: + +[`fuzz_config.py`](./fuzz-targets/fuzz_config.py) was migrated to this repository from the OSS-Fuzz project's repository +where it was originally created. As such, [`fuzz_config.py`](./fuzz-targets/fuzz_config.py) retains its original license +and copyright notice (Apache License, Version 2.0 and Copyright 2023 Google LLC respectively) as in a header +comment, followed by a notice stating that it has have been modified contributors to GitPython. +[LICENSE-APACHE](./LICENSE-APACHE) contains the original license used by the OSS-Fuzz project repository at the time the +file was migrated. [oss-fuzz-repo]: https://github.com/google/oss-fuzz diff --git a/fuzzing/dictionaries/fuzz_tree.dict b/fuzzing/dictionaries/fuzz_tree.dict deleted file mode 100644 index 3ebe52b7f..000000000 --- a/fuzzing/dictionaries/fuzz_tree.dict +++ /dev/null @@ -1,13 +0,0 @@ -"\\001\\000\\000\\000" -"_join_multiline_va" -"setdef" -"1\\000\\000\\000\\000\\000\\000\\000" -"\\000\\000\\000\\000\\000\\000\\000\\020" -"\\377\\377\\377\\377\\377\\377\\377r" -"\\001\\000\\000\\000\\000\\000\\000\\001" -"\\000\\000\\000\\000\\000\\000\\000\\014" -"\\000\\000\\000\\000\\000\\000\\000\\003" -"\\001\\000" -"\\032\\000\\000\\000\\000\\000\\000\\000" -"-\\000\\000\\000\\000\\000\\000\\000" -"__format" diff --git a/fuzzing/fuzz-targets/fuzz_repo.py b/fuzzing/fuzz-targets/fuzz_repo.py new file mode 100644 index 000000000..7bd82c120 --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_repo.py @@ -0,0 +1,47 @@ +import atheris +import io +import sys +import os +import tempfile + +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + +with atheris.instrument_imports(): + import git + + +def TestOneInput(data): + fdp = atheris.FuzzedDataProvider(data) + + with tempfile.TemporaryDirectory() as temp_dir: + repo = git.Repo.init(path=temp_dir) + + # Generate a minimal set of files based on fuzz data to minimize I/O operations. + file_paths = [os.path.join(temp_dir, f"File{i}") for i in range(min(3, fdp.ConsumeIntInRange(1, 3)))] + for file_path in file_paths: + with open(file_path, "wb") as f: + # The chosen upperbound for count of bytes we consume by writing to these + # files is somewhat arbitrary and may be worth experimenting with if the + # fuzzer coverage plateaus. + f.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512))) + + repo.index.add(file_paths) + repo.index.commit(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 80))) + + fuzz_tree = git.Tree(repo, git.Tree.NULL_BIN_SHA, 0, "") + + try: + fuzz_tree._deserialize(io.BytesIO(data)) + except IndexError: + return -1 + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/fuzz-targets/fuzz_tree.py b/fuzzing/fuzz-targets/fuzz_tree.py deleted file mode 100644 index 4e2038add..000000000 --- a/fuzzing/fuzz-targets/fuzz_tree.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -############################################################################### -# Note: This file has been modified by contributors to GitPython. -# The original state of this file may be referenced here: -# https://github.com/google/oss-fuzz/commit/f26f254558fc48f3c9bc130b10507386b94522da -############################################################################### -import atheris -import io -import sys -import os -import shutil - -if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): - path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) - os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary - -with atheris.instrument_imports(): - import git - - -def TestOneInput(data): - fdp = atheris.FuzzedDataProvider(data) - git_dir = "/tmp/.git" - head_file = os.path.join(git_dir, "HEAD") - refs_dir = os.path.join(git_dir, "refs") - common_dir = os.path.join(git_dir, "commondir") - objects_dir = os.path.join(git_dir, "objects") - - if os.path.isdir(git_dir): - shutil.rmtree(git_dir) - - os.mkdir(git_dir) - with open(head_file, "w") as f: - f.write(fdp.ConsumeUnicodeNoSurrogates(1024)) - os.mkdir(refs_dir) - os.mkdir(common_dir) - os.mkdir(objects_dir) - - _repo = git.Repo("/tmp/") - - fuzz_tree = git.Tree(_repo, git.Tree.NULL_BIN_SHA, 0, "") - try: - fuzz_tree._deserialize(io.BytesIO(data)) - except IndexError: - return -1 - - -def main(): - atheris.Setup(sys.argv, TestOneInput) - atheris.Fuzz() - - -if __name__ == "__main__": - main() From 48abb1cbc138cd9c013369ea4608dd2fe5ca7a62 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Sat, 4 May 2024 14:40:26 -0400 Subject: [PATCH 1013/1392] Add git.Blob fuzz target Based on the `test_blob.py` unit test. --- fuzzing/dictionaries/fuzz_blob.dict | 1 + fuzzing/fuzz-targets/fuzz_blob.py | 36 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 fuzzing/dictionaries/fuzz_blob.dict create mode 100644 fuzzing/fuzz-targets/fuzz_blob.py diff --git a/fuzzing/dictionaries/fuzz_blob.dict b/fuzzing/dictionaries/fuzz_blob.dict new file mode 100644 index 000000000..7f123f830 --- /dev/null +++ b/fuzzing/dictionaries/fuzz_blob.dict @@ -0,0 +1 @@ +"\\377\\377\\377\\377\\377\\377\\377\\377" diff --git a/fuzzing/fuzz-targets/fuzz_blob.py b/fuzzing/fuzz-targets/fuzz_blob.py new file mode 100644 index 000000000..9d296de40 --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_blob.py @@ -0,0 +1,36 @@ +import atheris +import sys +import os +import tempfile + +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + +with atheris.instrument_imports(): + import git + + +def TestOneInput(data): + fdp = atheris.FuzzedDataProvider(data) + + with tempfile.TemporaryDirectory() as temp_dir: + repo = git.Repo.init(path=temp_dir) + blob = git.Blob( + repo, + **{ + "binsha": git.Blob.NULL_BIN_SHA, + "path": fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes()), + }, + ) + + _ = blob.mime_type + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() From 6823e4543f33eb623df14a5a27c9731199de7a4f Mon Sep 17 00:00:00 2001 From: David Lakin Date: Sat, 4 May 2024 15:44:23 -0400 Subject: [PATCH 1014/1392] Use fuzzed data for all git.Blob arguments This increases the edges reached by the fuzzer, making for a more effective test with higher coverage. --- fuzzing/fuzz-targets/fuzz_blob.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/fuzzing/fuzz-targets/fuzz_blob.py b/fuzzing/fuzz-targets/fuzz_blob.py index 9d296de40..ce888e85f 100644 --- a/fuzzing/fuzz-targets/fuzz_blob.py +++ b/fuzzing/fuzz-targets/fuzz_blob.py @@ -16,13 +16,17 @@ def TestOneInput(data): with tempfile.TemporaryDirectory() as temp_dir: repo = git.Repo.init(path=temp_dir) - blob = git.Blob( - repo, - **{ - "binsha": git.Blob.NULL_BIN_SHA, - "path": fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes()), - }, - ) + binsha = fdp.ConsumeBytes(20) + mode = fdp.ConsumeInt(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())) + path = fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes()) + + try: + blob = git.Blob(repo, binsha, mode, path) + except AssertionError as e: + if "Require 20 byte binary sha, got" in str(e): + return -1 + else: + raise e _ = blob.mime_type From e15caab8e70adc44b796bd3d972e1d34d30ad7ee Mon Sep 17 00:00:00 2001 From: Jirka Date: Tue, 7 May 2024 19:26:54 +0200 Subject: [PATCH 1015/1392] lint: switch order Ruff's hooks `fix` -> `format` --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 585b4f04d..987d86cd9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,12 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.2 + rev: v0.4.3 hooks: - - id: ruff-format - exclude: ^git/ext/ - id: ruff args: ["--fix"] exclude: ^git/ext/ + - id: ruff-format + exclude: ^git/ext/ - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.9.0.6 From 2cfd2007b4a73bb061506e7c521570e9a0ec3f96 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Wed, 8 May 2024 03:20:18 -0400 Subject: [PATCH 1016/1392] Update OSS-Fuzz Scripts to Use New QA-Assets Repo Structure This change is required to support the changes to the seed data repo structure introduced in: https://github.com/gitpython-developers/qa-assets/pull/2 This moves most of the seed data related build steps into the OSS-Fuzz Docker image build via `container-environment-bootstrap.sh`. This includes moveing the dictionaries into that repo. The fuzzing/README.md here should be updated in a follow-up with a link to the qa-assets repo (and probably some context setting about corpora in general) but I have opted to defer that as I think the functionality added by the seed data improvements is valuable as is and shouldn't be blocked by documentation writers block. --- fuzzing/README.md | 19 ------ fuzzing/dictionaries/fuzz_blob.dict | 1 - fuzzing/dictionaries/fuzz_config.dict | 56 ---------------- fuzzing/oss-fuzz-scripts/build.sh | 27 +------- .../container-environment-bootstrap.sh | 64 +++++++++++++++---- 5 files changed, 53 insertions(+), 114 deletions(-) delete mode 100644 fuzzing/dictionaries/fuzz_blob.dict delete mode 100644 fuzzing/dictionaries/fuzz_config.dict diff --git a/fuzzing/README.md b/fuzzing/README.md index 9d02bf72f..286f529eb 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -76,25 +76,6 @@ Contains Python files for each fuzz test. reason, fuzz tests should gracefully handle anticipated exception cases with a `try`/`except` block to avoid false positives that halt the fuzzing engine. -### Dictionaries (`dictionaries/`) - -Provides hints to the fuzzing engine about inputs that might trigger unique code paths. Each fuzz target may have a -corresponding `.dict` file. For information about dictionary syntax, refer to -the [LibFuzzer documentation on the subject](https://llvm.org/docs/LibFuzzer.html#dictionaries). - -**Things to Know**: - -- OSS-Fuzz loads dictionary files per fuzz target if one exists with the same name, all others are ignored. -- Most entries in the dictionary files found here are escaped hex or Unicode values that were recommended by the fuzzing - engine after previous runs. -- A default set of dictionary entries are created for all fuzz targets as part of the build process, regardless of an - existing file here. -- Development or updates to dictionaries should reflect the varied formats and edge cases relevant to the - functionalities under test. -- Example dictionaries (some of which are used to build the default dictionaries mentioned above) can be found here: - - [AFL++ dictionary repository](https://github.com/AFLplusplus/AFLplusplus/tree/stable/dictionaries#readme) - - [Google/fuzzing dictionary repository](https://github.com/google/fuzzing/tree/master/dictionaries) - ### OSS-Fuzz Scripts (`oss-fuzz-scripts/`) Includes scripts for building and integrating fuzz targets with OSS-Fuzz: diff --git a/fuzzing/dictionaries/fuzz_blob.dict b/fuzzing/dictionaries/fuzz_blob.dict deleted file mode 100644 index 7f123f830..000000000 --- a/fuzzing/dictionaries/fuzz_blob.dict +++ /dev/null @@ -1 +0,0 @@ -"\\377\\377\\377\\377\\377\\377\\377\\377" diff --git a/fuzzing/dictionaries/fuzz_config.dict b/fuzzing/dictionaries/fuzz_config.dict deleted file mode 100644 index b545ddfc8..000000000 --- a/fuzzing/dictionaries/fuzz_config.dict +++ /dev/null @@ -1,56 +0,0 @@ -"\\004\\000\\000\\000\\000\\000\\000\\000" -"\\006\\000\\000\\000\\000\\000\\000\\000" -"_validate_value_" -"\\000\\000\\000\\000\\000\\000\\000\\000" -"rem" -"__eq__" -"\\001\\000\\000\\000" -"__abstrac" -"_mutating_methods_" -"items" -"\\0021\\"" -"\\001\\000" -"\\000\\000\\000\\000" -"DEFAULT" -"getfloat" -"\\004\\000\\000\\000\\000\\000\\000\\000" -"news" -"\\037\\000\\000\\000\\000\\000\\000\\000" -"\\001\\000\\000\\000\\000\\000\\000\\037" -"\\000\\000\\000\\000\\000\\000\\000\\014" -"list" -"\\376\\377\\377\\377\\377\\377\\377\\377" -"items_all" -"\\004\\000\\000\\000\\000\\000\\000\\000" -"\\377\\377\\377\\377\\377\\377\\377\\014" -"\\001\\000\\000\\000" -"_acqui" -"\\000\\000\\000\\000\\000\\000\\000\\000" -"__ne__" -"__exit__" -"__modu" -"uucp" -"__str__" -"\\001\\000\\000\\000" -"\\017\\000\\000\\000\\000\\000\\000\\000" -"_has_incl" -"update" -"\\377\\377\\377\\377\\377\\377\\377\\023" -"setdef" -"setdefaul" -"\\000\\000\\000\\000" -"\\001\\000\\000\\000" -"\\001\\000" -"\\022\\000\\000\\000\\000\\000\\000\\000" -"_value_to_string" -"__abstr" -"\\001\\000\\000\\000\\000\\000\\000\\000" -"\\000\\000\\000\\000\\000\\000\\000\\022" -"\\377\\377\\377\\377" -"\\004\\000\\000\\000\\000\\000\\000\\000" -"\\000\\000\\000\\000\\000\\000\\000\\000" -"\\000\\000\\000\\000\\000\\000\\000\\037" -"\\001\\000\\000\\000\\000\\000\\000\\013" -"_OPT_TM" -"__name__" -"_get_conv" diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh index 58c9adb5a..e0b3a50ab 100644 --- a/fuzzing/oss-fuzz-scripts/build.sh +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -7,34 +7,13 @@ set -euo pipefail python3 -m pip install . -# Directory to look in for dictionaries, options files, and seed corpora: -SEED_DATA_DIR="$SRC/seed_data" - -find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name '*.dict' \) \ - ! \( -name '__base.*' \) -exec printf 'Copying: %s\n' {} \; \ +find "$SRC" -maxdepth 1 \ + \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name '*.dict' \) \ + -exec printf '[%s] Copying: %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" {} \; \ -exec chmod a-x {} \; \ -exec cp {} "$OUT" \; # Build fuzzers in $OUT. find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):." - - common_base_dictionary_filename="$SEED_DATA_DIR/__base.dict" - if [[ -r "$common_base_dictionary_filename" ]]; then - # Strip the `.py` extension from the filename and replace it with `.dict`. - fuzz_harness_dictionary_filename="$(basename "$fuzz_harness" .py).dict" - output_file="$OUT/$fuzz_harness_dictionary_filename" - - printf 'Appending %s to %s\n' "$common_base_dictionary_filename" "$output_file" - if [[ -s "$output_file" ]]; then - # If a dictionary file for this fuzzer already exists and is not empty, - # we append a new line to the end of it before appending any new entries. - # - # LibFuzzer will happily ignore multiple empty lines in a dictionary but fail with an error - # if any single line has incorrect syntax (e.g., if we accidentally add two entries to the same line.) - # See docs for valid syntax: https://llvm.org/docs/LibFuzzer.html#id32 - echo >>"$output_file" - fi - cat "$common_base_dictionary_filename" >>"$output_file" - fi done diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh index 76ec97c7f..bbdcf5357 100755 --- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -9,23 +9,20 @@ set -euo pipefail # Prerequisites # ################# -for cmd in python3 git wget rsync; do +for cmd in python3 git wget zip; do command -v "$cmd" >/dev/null 2>&1 || { printf '[%s] Required command %s not found, exiting.\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$cmd" >&2 exit 1 } done -SEED_DATA_DIR="$SRC/seed_data" -mkdir -p "$SEED_DATA_DIR" - ############# # Functions # ############# download_and_concatenate_common_dictionaries() { # Assign the first argument as the target file where all contents will be concatenated - target_file="$1" + local target_file="$1" # Shift the arguments so the first argument (target_file path) is removed # and only URLs are left for the loop below. @@ -38,22 +35,61 @@ download_and_concatenate_common_dictionaries() { done } -fetch_seed_corpora() { - # Seed corpus zip files are hosted in a separate repository to avoid additional bloat in this repo. - git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git qa-assets && - rsync -avc qa-assets/gitpython/corpra/ "$SEED_DATA_DIR/" && - rm -rf qa-assets # Clean up the cloned repo to keep the Docker image as slim as possible. +create_seed_corpora_zips() { + local seed_corpora_dir="$1" + local output_zip + for dir in "$seed_corpora_dir"/*; do + if [ -d "$dir" ] && [ -n "$dir" ]; then + output_zip="$SRC/$(basename "$dir")_seed_corpus.zip" + printf '[%s] Zipping the contents of %s into %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$dir" "$output_zip" + zip -jur "$output_zip" "$dir"/* + fi + done +} + +prepare_dictionaries_for_fuzz_targets() { + local dictionaries_dir="$1" + local fuzz_targets_dir="$2" + local common_base_dictionary_filename="$WORK/__base.dict" + + printf '[%s] Copying .dict files from %s to %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$dictionaries_dir" "$SRC/" + cp -v "$dictionaries_dir"/*.dict "$SRC/" + + download_and_concatenate_common_dictionaries "$common_base_dictionary_filename" \ + "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \ + "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict" + + find "$fuzz_targets_dir" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do + if [[ -r "$common_base_dictionary_filename" ]]; then + # Strip the `.py` extension from the filename and replace it with `.dict`. + fuzz_harness_dictionary_filename="$(basename "$fuzz_harness" .py).dict" + local output_file="$SRC/$fuzz_harness_dictionary_filename" + + printf '[%s] Appending %s to %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$common_base_dictionary_filename" "$output_file" + if [[ -s "$output_file" ]]; then + # If a dictionary file for this fuzzer already exists and is not empty, + # we append a new line to the end of it before appending any new entries. + # + # LibFuzzer will happily ignore multiple empty lines in a dictionary but fail with an error + # if any single line has incorrect syntax (e.g., if we accidentally add two entries to the same line.) + # See docs for valid syntax: https://llvm.org/docs/LibFuzzer.html#id32 + echo >>"$output_file" + fi + cat "$common_base_dictionary_filename" >>"$output_file" + fi + done } ######################## # Main execution logic # ######################## +# Seed corpora and dictionaries are hosted in a separate repository to avoid additional bloat in this repo. +# We clone into the $WORK directory because OSS-Fuzz cleans it up after building the image, keeping the image small. +git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git "$WORK/qa-assets" -fetch_seed_corpora +create_seed_corpora_zips "$WORK/qa-assets/gitpython/corpora" -download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \ - "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \ - "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict" +prepare_dictionaries_for_fuzz_targets "$WORK/qa-assets/gitpython/dictionaries" "$SRC/gitpython/fuzzing" # The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below. python3 -m pip install --upgrade pip From a915adf08e570d8989bb070f647e2a3ee941871d Mon Sep 17 00:00:00 2001 From: David Lakin Date: Wed, 8 May 2024 17:06:19 -0400 Subject: [PATCH 1017/1392] Add `Diff` Fuzz Target Adds a new `fuzz_diff.py` fuzz target that covers `Diff` class initialization using fuzzed data. --- fuzzing/fuzz-targets/fuzz_diff.py | 54 +++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 fuzzing/fuzz-targets/fuzz_diff.py diff --git a/fuzzing/fuzz-targets/fuzz_diff.py b/fuzzing/fuzz-targets/fuzz_diff.py new file mode 100644 index 000000000..cf01e7ffa --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_diff.py @@ -0,0 +1,54 @@ +import sys +import os +import tempfile +from binascii import Error as BinasciiError + +import atheris + +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + +with atheris.instrument_imports(): + from git import Repo, Diff + + +def TestOneInput(data): + fdp = atheris.FuzzedDataProvider(data) + + with tempfile.TemporaryDirectory() as temp_dir: + repo = Repo.init(path=temp_dir) + try: + Diff( + repo, + a_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + b_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + a_blob_id=fdp.ConsumeBytes(20), + b_blob_id=fdp.ConsumeBytes(20), + a_mode=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + b_mode=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + new_file=fdp.ConsumeBool(), + deleted_file=fdp.ConsumeBool(), + copied_file=fdp.ConsumeBool(), + raw_rename_from=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + raw_rename_to=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + diff=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + change_type=fdp.PickValueInList(["A", "D", "C", "M", "R", "T", "U"]), + score=fdp.ConsumeIntInRange(0, fdp.remaining_bytes()), + ) + except BinasciiError: + return -1 + except AssertionError as e: + if "Require 20 byte binary sha, got" in str(e): + return -1 + else: + raise e + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() From 989ae1ac03e25a5ce51d4c615128dcf75b9e24f5 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Wed, 8 May 2024 19:28:29 -0400 Subject: [PATCH 1018/1392] Read class properties & call methods to cover more features Property access and private methods on the `Diff` class are complex and involve encoding and decoding operations that warrant being tested. This test borrows its design from the `test_diff.py` unit test file. --- fuzzing/fuzz-targets/fuzz_diff.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/fuzzing/fuzz-targets/fuzz_diff.py b/fuzzing/fuzz-targets/fuzz_diff.py index cf01e7ffa..ba44995f2 100644 --- a/fuzzing/fuzz-targets/fuzz_diff.py +++ b/fuzzing/fuzz-targets/fuzz_diff.py @@ -1,5 +1,6 @@ import sys import os +import io import tempfile from binascii import Error as BinasciiError @@ -13,13 +14,26 @@ from git import Repo, Diff +class BytesProcessAdapter: + """Allows bytes to be used as process objects returned by subprocess.Popen.""" + + def __init__(self, input_string): + self.stdout = io.BytesIO(input_string) + self.stderr = io.BytesIO() + + def wait(self): + return 0 + + poll = wait + + def TestOneInput(data): fdp = atheris.FuzzedDataProvider(data) with tempfile.TemporaryDirectory() as temp_dir: repo = Repo.init(path=temp_dir) try: - Diff( + diff = Diff( repo, a_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), b_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), @@ -44,6 +58,21 @@ def TestOneInput(data): else: raise e + _ = diff.__str__() + _ = diff.a_path + _ = diff.b_path + _ = diff.rename_from + _ = diff.rename_to + _ = diff.renamed_file + + diff_index = diff._index_from_patch_format( + repo, proc=BytesProcessAdapter(fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes()))) + ) + + diff._handle_diff_line( + lines_bytes=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), repo=repo, index=diff_index + ) + def main(): atheris.Setup(sys.argv, TestOneInput) From 315a2fd03c94c93d4a7089d23d734e4aaccbe066 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Wed, 15 May 2024 13:36:29 -0400 Subject: [PATCH 1019/1392] Instrument test utility functions to increase fuzzer efficiency Fuzz Introspector was reporting a high percentage of fuzz blockers in the `fuzz_diff` test. This means the fuzzing engine was unable to gain visibility into functions lower in the call stack than the blocking functions, making it less effective at producing interesting input data. This clears a large percentage of the fuzz blockers by adding fuzzer instrumentation to them via the `@atheris.instrument_func` decorator. --- fuzzing/fuzz-targets/fuzz_diff.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fuzzing/fuzz-targets/fuzz_diff.py b/fuzzing/fuzz-targets/fuzz_diff.py index ba44995f2..d4bd68b57 100644 --- a/fuzzing/fuzz-targets/fuzz_diff.py +++ b/fuzzing/fuzz-targets/fuzz_diff.py @@ -17,16 +17,19 @@ class BytesProcessAdapter: """Allows bytes to be used as process objects returned by subprocess.Popen.""" + @atheris.instrument_func def __init__(self, input_string): self.stdout = io.BytesIO(input_string) self.stderr = io.BytesIO() + @atheris.instrument_func def wait(self): return 0 poll = wait +@atheris.instrument_func def TestOneInput(data): fdp = atheris.FuzzedDataProvider(data) From cf81c6c98155c24d69af6f1a8eca368ad1a5d962 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 26 May 2024 16:40:23 -0400 Subject: [PATCH 1020/1392] Momentarily downgrade Git on Cygwin to investigate failures Using this older version is not in general secure, since the new version is a security update. It is sometimes acceptable to run software with security bugs in CI workflows, but the intent of this change is just to check if the version of the Cygwin `git` package is the cause of the failures. If so, they can probably be fixed or worked around in a better way than downgrading. (Furthermore, the lower version of the `git` package will not always be avaialable from Cygwin's repositories.) --- .github/workflows/cygwin-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 61e6a3089..a2fe588ad 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -30,7 +30,7 @@ jobs: - name: Set up Cygwin uses: egor-tensin/setup-cygwin@v4 with: - packages: python39=3.9.16-1 python39-pip python39-virtualenv git + packages: python39=3.9.16-1 python39-pip python39-virtualenv git=2.43.0-1 - name: Arrange for verbose output run: | From eb06a18d83eda0ae04e2a00b2d656da147e9188a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 26 May 2024 16:45:57 -0400 Subject: [PATCH 1021/1392] Unpin Cygwin `git`; add our `.git` as a `safe.directory` This undoes the change of pinning Git to an earlier version (before the recent security update) on Cygwin, and instead adds the `.git` subdirectory of the `GitPython` directory as an additional value of the multi-valued `safe.directory` Git configuration variable. --- .github/workflows/cygwin-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index a2fe588ad..bde4ea659 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -30,7 +30,7 @@ jobs: - name: Set up Cygwin uses: egor-tensin/setup-cygwin@v4 with: - packages: python39=3.9.16-1 python39-pip python39-virtualenv git=2.43.0-1 + packages: python39=3.9.16-1 python39-pip python39-virtualenv git - name: Arrange for verbose output run: | @@ -40,6 +40,7 @@ jobs: - name: Special configuration for Cygwin git run: | git config --global --add safe.directory "$(pwd)" + git config --global --add safe.directory "$(pwd)/.git" git config --global core.autocrlf false - name: Prepare this repo for tests From d3b181d54a8da6b8561474dba1333682b47b7ba7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 13:22:29 +0000 Subject: [PATCH 1022/1392] Bump Vampire/setup-wsl from 3.0.0 to 3.1.0 Bumps [Vampire/setup-wsl](https://github.com/vampire/setup-wsl) from 3.0.0 to 3.1.0. - [Release notes](https://github.com/vampire/setup-wsl/releases) - [Commits](https://github.com/vampire/setup-wsl/compare/v3.0.0...v3.1.0) --- updated-dependencies: - dependency-name: Vampire/setup-wsl dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4c918a92d..574048620 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -44,7 +44,7 @@ jobs: - name: Set up WSL (Windows) if: startsWith(matrix.os, 'windows') - uses: Vampire/setup-wsl@v3.0.0 + uses: Vampire/setup-wsl@v3.1.0 with: distribution: Debian From 7bdcfa556ad476d89a3643137e97ee5749e3c7df Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 27 May 2024 21:27:07 +0200 Subject: [PATCH 1023/1392] Update to the fixed version of `Vampire` --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 574048620..031b0e6b2 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -44,7 +44,7 @@ jobs: - name: Set up WSL (Windows) if: startsWith(matrix.os, 'windows') - uses: Vampire/setup-wsl@v3.1.0 + uses: Vampire/setup-wsl@v3.1.1 with: distribution: Debian From 6d52bdbe6a546ecb76e28f7dde45b44fe8577010 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Wed, 29 May 2024 22:06:40 -0400 Subject: [PATCH 1024/1392] Add Submodules Fuzz Target Fuzz Introspector heuristics suggest the Submodule API code represent "optimal analysis targets" that should yield a meaningful increase in code coverage. The changes here introduce a first pass at implementing a fuzz harness that cover the primary APIs/methods related to Submodules. Of particular interest to me is the `Submodule.config_writer()` coverage. Please note however, there is likely plenty of room for improvement in this harness in terms of both code coverage as well as performance; the latter of which will see significant benefit from a well curated seed corpus of `.gitmodules` file like inputs. The `ParsingError` raised by the fuzzer without a good seed corpus hinders test efficacy significantly. --- fuzzing/fuzz-targets/fuzz_submodule.py | 93 ++++++++++++++++++++++++++ fuzzing/fuzz-targets/utils.py | 22 ++++++ 2 files changed, 115 insertions(+) create mode 100644 fuzzing/fuzz-targets/fuzz_submodule.py create mode 100644 fuzzing/fuzz-targets/utils.py diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py new file mode 100644 index 000000000..ddcbaa00f --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -0,0 +1,93 @@ +import atheris +import sys +import os +import tempfile +from configparser import ParsingError +from utils import is_expected_exception_message + +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + +with atheris.instrument_imports(): + from git import Repo, GitCommandError, InvalidGitRepositoryError + + +def TestOneInput(data): + fdp = atheris.FuzzedDataProvider(data) + + with tempfile.TemporaryDirectory() as repo_temp_dir: + repo = Repo.init(path=repo_temp_dir) + repo.index.commit("Initial commit") + + try: + with tempfile.TemporaryDirectory() as submodule_temp_dir: + sub_repo = Repo.init(submodule_temp_dir, bare=fdp.ConsumeBool()) + sub_repo.index.commit(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))) + + submodule_name = f"submodule_{fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))}" + submodule_path = os.path.join(repo.working_tree_dir, submodule_name) + submodule_url = sub_repo.git_dir + + submodule = repo.create_submodule(submodule_name, submodule_path, url=submodule_url) + repo.index.commit(f"Added submodule {submodule_name}") + + with submodule.config_writer() as writer: + key_length = fdp.ConsumeIntInRange(1, max(1, fdp.remaining_bytes())) + value_length = fdp.ConsumeIntInRange(1, max(1, fdp.remaining_bytes())) + + writer.set_value( + fdp.ConsumeUnicodeNoSurrogates(key_length), fdp.ConsumeUnicodeNoSurrogates(value_length) + ) + writer.release() + + submodule.update(init=fdp.ConsumeBool(), dry_run=fdp.ConsumeBool(), force=fdp.ConsumeBool()) + + submodule_repo = submodule.module() + new_file_path = os.path.join( + submodule_repo.working_tree_dir, + f"new_file_{fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))}", + ) + with open(new_file_path, "wb") as new_file: + new_file.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512))) + submodule_repo.index.add([new_file_path]) + submodule_repo.index.commit("Added new file to submodule") + + repo.submodule_update(recursive=fdp.ConsumeBool()) + submodule_repo.head.reset(commit="HEAD~1", working_tree=fdp.ConsumeBool(), head=fdp.ConsumeBool()) + # Use fdp.PickValueInList to ensure at least one of 'module' or 'configuration' is True + module_option_value, configuration_option_value = fdp.PickValueInList( + [(True, False), (False, True), (True, True)] + ) + submodule.remove( + module=module_option_value, + configuration=configuration_option_value, + dry_run=fdp.ConsumeBool(), + force=fdp.ConsumeBool(), + ) + repo.index.commit(f"Removed submodule {submodule_name}") + + except (ParsingError, GitCommandError, InvalidGitRepositoryError, FileNotFoundError, BrokenPipeError): + return -1 + except (ValueError, OSError) as e: + expected_messages = [ + "SHA is empty", + "Reference at", + "embedded null byte", + "This submodule instance does not exist anymore", + "cmd stdin was empty", + "File name too long", + ] + if is_expected_exception_message(e, expected_messages): + return -1 + else: + raise e + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/fuzz-targets/utils.py b/fuzzing/fuzz-targets/utils.py new file mode 100644 index 000000000..42faa8eb0 --- /dev/null +++ b/fuzzing/fuzz-targets/utils.py @@ -0,0 +1,22 @@ +import atheris # pragma: no cover +from typing import List # pragma: no cover + + +@atheris.instrument_func +def is_expected_exception_message(exception: Exception, error_message_list: List[str]) -> bool: # pragma: no cover + """ + Checks if the message of a given exception matches any of the expected error messages, case-insensitively. + + Args: + exception (Exception): The exception object raised during execution. + error_message_list (List[str]): A list of error message substrings to check against the exception's message. + + Returns: + bool: True if the exception's message contains any of the substrings from the error_message_list, + case-insensitively, otherwise False. + """ + exception_message = str(exception).lower() + for error in error_message_list: + if error.lower() in exception_message: + return True + return False From 9e67138819c7e081fee89a5b855c89b538a8f604 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 30 May 2024 12:22:45 +0200 Subject: [PATCH 1025/1392] precommit: enable `end-of-file-fixer` --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 585b4f04d..50f430084 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: + - id: end-of-file-fixer - id: check-toml - id: check-yaml - id: check-merge-conflict From 96e21f0055060b01d03b705cdb582edd3551aa43 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 30 May 2024 12:24:19 +0200 Subject: [PATCH 1026/1392] apply --- doc/source/index.rst | 1 - doc/source/intro.rst | 1 - doc/source/roadmap.rst | 1 - test/fixtures/.gitconfig | 2 +- test/fixtures/blame | 2 +- test/fixtures/cat_file_blob | 2 +- test/fixtures/git_config | 1 - test/fixtures/git_config_with_empty_value | 2 +- test/fixtures/rev_list_bisect_all | 1 - test/fixtures/rev_list_commit_diffs | 1 - test/fixtures/rev_list_commit_idabbrev | 1 - test/fixtures/rev_list_commit_stats | 1 - 12 files changed, 4 insertions(+), 12 deletions(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index 72db8ee5a..ca5229ac3 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -21,4 +21,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 4f22a0942..d053bd117 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -122,4 +122,3 @@ License Information =================== GitPython is licensed under the New BSD License. See the LICENSE file for more information. - diff --git a/doc/source/roadmap.rst b/doc/source/roadmap.rst index a573df33a..34c953626 100644 --- a/doc/source/roadmap.rst +++ b/doc/source/roadmap.rst @@ -6,4 +6,3 @@ The full list of milestones including associated tasks can be found on GitHub: https://github.com/gitpython-developers/GitPython/issues Select the respective milestone to filter the list of issues accordingly. - diff --git a/test/fixtures/.gitconfig b/test/fixtures/.gitconfig index 6a0459f6b..f6c25c15a 100644 --- a/test/fixtures/.gitconfig +++ b/test/fixtures/.gitconfig @@ -1,3 +1,3 @@ [alias] rbi = "!g() { git rebase -i origin/${1:-master} ; } ; g" - expush = "!f() { git branch -f tmp ; { git rbi $1 && git push ; } ; git reset --hard tmp ; git rebase origin/${1:-master}; } ; f" \ No newline at end of file + expush = "!f() { git branch -f tmp ; { git rbi $1 && git push ; } ; git reset --hard tmp ; git rebase origin/${1:-master}; } ; f" diff --git a/test/fixtures/blame b/test/fixtures/blame index 10c141dda..949976c5d 100644 --- a/test/fixtures/blame +++ b/test/fixtures/blame @@ -128,4 +128,4 @@ b6e1b765e0c15586a2c5b9832854f95defd71e1f 23 23 634396b2f541a9f2d58b00be1a07f0c358b999b3 11 24 2 VERSION = '1.0.0' 634396b2f541a9f2d58b00be1a07f0c358b999b3 12 25 - end \ No newline at end of file + end diff --git a/test/fixtures/cat_file_blob b/test/fixtures/cat_file_blob index 70c379b63..802992c42 100644 --- a/test/fixtures/cat_file_blob +++ b/test/fixtures/cat_file_blob @@ -1 +1 @@ -Hello world \ No newline at end of file +Hello world diff --git a/test/fixtures/git_config b/test/fixtures/git_config index a8cad56e8..d3066d86e 100644 --- a/test/fixtures/git_config +++ b/test/fixtures/git_config @@ -43,4 +43,3 @@ # inclusions should be processed immediately [sec] var1 = value1_main - diff --git a/test/fixtures/git_config_with_empty_value b/test/fixtures/git_config_with_empty_value index 0427caea5..83de84c8b 100644 --- a/test/fixtures/git_config_with_empty_value +++ b/test/fixtures/git_config_with_empty_value @@ -1,4 +1,4 @@ [color] ui [core] - filemode = true \ No newline at end of file + filemode = true diff --git a/test/fixtures/rev_list_bisect_all b/test/fixtures/rev_list_bisect_all index 342ea94ae..60d382d01 100644 --- a/test/fixtures/rev_list_bisect_all +++ b/test/fixtures/rev_list_bisect_all @@ -48,4 +48,3 @@ committer David Aguilar 1220418344 -0700 This resolves the issue mentioned in that thread. Signed-off-by: David Aguilar - diff --git a/test/fixtures/rev_list_commit_diffs b/test/fixtures/rev_list_commit_diffs index 20397e2e4..c39df2061 100644 --- a/test/fixtures/rev_list_commit_diffs +++ b/test/fixtures/rev_list_commit_diffs @@ -5,4 +5,3 @@ author Tom Preston-Werner 1193200199 -0700 committer Tom Preston-Werner 1193200199 -0700 fix some initialization warnings - diff --git a/test/fixtures/rev_list_commit_idabbrev b/test/fixtures/rev_list_commit_idabbrev index 9385ba713..6266df93e 100644 --- a/test/fixtures/rev_list_commit_idabbrev +++ b/test/fixtures/rev_list_commit_idabbrev @@ -5,4 +5,3 @@ author tom 1195608462 -0800 committer tom 1195608462 -0800 fix tests on other machines - diff --git a/test/fixtures/rev_list_commit_stats b/test/fixtures/rev_list_commit_stats index 60aa8cf58..c78aadeb5 100644 --- a/test/fixtures/rev_list_commit_stats +++ b/test/fixtures/rev_list_commit_stats @@ -4,4 +4,3 @@ author Tom Preston-Werner 1191997100 -0700 committer Tom Preston-Werner 1191997100 -0700 initial grit setup - From 2ce9675b7238fb1a498f1ea4f5dae8a26d4b89ec Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 30 May 2024 12:25:33 +0200 Subject: [PATCH 1027/1392] precommit: enable `validate-pyproject` --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 585b4f04d..02950db8c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,3 +21,8 @@ repos: - id: check-toml - id: check-yaml - id: check-merge-conflict + +- repo: https://github.com/abravalheri/validate-pyproject + rev: v0.16 + hooks: + - id: validate-pyproject \ No newline at end of file From 7b684cd43cf0f9c54adb8a8def54fa07f5cfd145 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 30 May 2024 10:34:49 -0400 Subject: [PATCH 1028/1392] Add graceful handling of expected exceptions in `fuzz_submodule.py` Fixes: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=69350 **`IsADirectoryError`** Fuzzer provided input data can sometimes produce filenames that look like directories and raise `IsADirectoryError` exceptions which crash the fuzzer. This commit catches those cases and returns -1 to instruct libfuzzer that the inputs are not valuable to add to the corpus. **`FileExistsError`** Similar to the above, this is a possible exception case produced by the fuzzed data and not a bug so its handled the same. --- fuzzing/fuzz-targets/fuzz_submodule.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index ddcbaa00f..9406fc68b 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -67,7 +67,15 @@ def TestOneInput(data): ) repo.index.commit(f"Removed submodule {submodule_name}") - except (ParsingError, GitCommandError, InvalidGitRepositoryError, FileNotFoundError, BrokenPipeError): + except ( + ParsingError, + GitCommandError, + InvalidGitRepositoryError, + FileNotFoundError, + FileExistsError, + IsADirectoryError, + BrokenPipeError, + ): return -1 except (ValueError, OSError) as e: expected_messages = [ From 6c00ce602eb19eda342e827a25d005610ce92fa8 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 30 May 2024 13:53:42 -0400 Subject: [PATCH 1029/1392] Improve file name generation to prevent "File name too long" `OSError`'s Adds a utility function to limit the maximum file name legnth produced by the fuzzer to a max size dictated by the host its run on. --- fuzzing/fuzz-targets/fuzz_submodule.py | 13 ++++++------- fuzzing/fuzz-targets/utils.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index 9406fc68b..817ce8f98 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -3,7 +3,7 @@ import os import tempfile from configparser import ParsingError -from utils import is_expected_exception_message +from utils import is_expected_exception_message, get_max_filename_length if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) @@ -42,12 +42,12 @@ def TestOneInput(data): writer.release() submodule.update(init=fdp.ConsumeBool(), dry_run=fdp.ConsumeBool(), force=fdp.ConsumeBool()) - submodule_repo = submodule.module() - new_file_path = os.path.join( - submodule_repo.working_tree_dir, - f"new_file_{fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))}", + + new_file_name = fdp.ConsumeUnicodeNoSurrogates( + fdp.ConsumeIntInRange(1, max(1, get_max_filename_length(submodule_repo.working_tree_dir))) ) + new_file_path = os.path.join(submodule_repo.working_tree_dir, new_file_name) with open(new_file_path, "wb") as new_file: new_file.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512))) submodule_repo.index.add([new_file_path]) @@ -77,14 +77,13 @@ def TestOneInput(data): BrokenPipeError, ): return -1 - except (ValueError, OSError) as e: + except ValueError as e: expected_messages = [ "SHA is empty", "Reference at", "embedded null byte", "This submodule instance does not exist anymore", "cmd stdin was empty", - "File name too long", ] if is_expected_exception_message(e, expected_messages): return -1 diff --git a/fuzzing/fuzz-targets/utils.py b/fuzzing/fuzz-targets/utils.py index 42faa8eb0..86f049341 100644 --- a/fuzzing/fuzz-targets/utils.py +++ b/fuzzing/fuzz-targets/utils.py @@ -1,4 +1,5 @@ import atheris # pragma: no cover +import os from typing import List # pragma: no cover @@ -20,3 +21,17 @@ def is_expected_exception_message(exception: Exception, error_message_list: List if error.lower() in exception_message: return True return False + + +@atheris.instrument_func +def get_max_filename_length(path: str) -> int: + """ + Get the maximum filename length for the filesystem containing the given path. + + Args: + path (str): The path to check the filesystem for. + + Returns: + int: The maximum filename length. + """ + return os.pathconf(path, "PC_NAME_MAX") From 2a2294f9d1e46d9bbe11cd2031d62e5441fe19c4 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 30 May 2024 14:02:27 -0400 Subject: [PATCH 1030/1392] Improve `fuzz_submodule.py` coverage & efficacy The fuzzer was having trouble analyzing `fuzz_submodule.py` when using the `atheris.instrument_imports()` context manager. Switching to `atheris.instrument_all()` instead slightly increases the startup time for the fuzzer, but significantly improves the fuzzing engines ability to identify new coverage. The changes here also disable warnings that are logged to `stdout` from the SUT. These warnings are expected to happen with some inputs and clutter the fuzzer output logs. They can be optionally re-enabled for debugging by passing a flag o the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable. --- fuzzing/fuzz-targets/fuzz_submodule.py | 16 +++++++++++++--- fuzzing/fuzz-targets/utils.py | 4 ++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index 817ce8f98..53f5a7884 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -4,13 +4,22 @@ import tempfile from configparser import ParsingError from utils import is_expected_exception_message, get_max_filename_length +from git import Repo, GitCommandError, InvalidGitRepositoryError -if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # pragma: no cover path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary -with atheris.instrument_imports(): - from git import Repo, GitCommandError, InvalidGitRepositoryError +if not sys.warnoptions: # pragma: no cover + # The warnings filter below can be overridden by passing the -W option + # to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable. + import warnings + import logging + + # Fuzzing data causes some plugins to generate a large number of warnings + # which are not usually interesting and make the test output hard to read, so we ignore them. + warnings.simplefilter("ignore") + logging.getLogger().setLevel(logging.ERROR) def TestOneInput(data): @@ -92,6 +101,7 @@ def TestOneInput(data): def main(): + atheris.instrument_all() atheris.Setup(sys.argv, TestOneInput) atheris.Fuzz() diff --git a/fuzzing/fuzz-targets/utils.py b/fuzzing/fuzz-targets/utils.py index 86f049341..f522d2959 100644 --- a/fuzzing/fuzz-targets/utils.py +++ b/fuzzing/fuzz-targets/utils.py @@ -1,5 +1,5 @@ import atheris # pragma: no cover -import os +import os # pragma: no cover from typing import List # pragma: no cover @@ -24,7 +24,7 @@ def is_expected_exception_message(exception: Exception, error_message_list: List @atheris.instrument_func -def get_max_filename_length(path: str) -> int: +def get_max_filename_length(path: str) -> int: # pragma: no cover """ Get the maximum filename length for the filesystem containing the given path. From 57a56a8a2874d2ab76f4034b9d3c98e09ed7fa35 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 30 May 2024 14:12:02 -0400 Subject: [PATCH 1031/1392] Add graceful handling for `NotADirectoryError`s --- fuzzing/fuzz-targets/fuzz_submodule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index 53f5a7884..cfd1a6d3f 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -83,6 +83,7 @@ def TestOneInput(data): FileNotFoundError, FileExistsError, IsADirectoryError, + NotADirectoryError, BrokenPipeError, ): return -1 From 2b64dee466ed72523684f90a037d604355121df0 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 30 May 2024 14:17:20 -0400 Subject: [PATCH 1032/1392] Improve comment wording --- fuzzing/fuzz-targets/fuzz_submodule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index cfd1a6d3f..92b569949 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -16,7 +16,7 @@ import warnings import logging - # Fuzzing data causes some plugins to generate a large number of warnings + # Fuzzing data causes some modules to generate a large number of warnings # which are not usually interesting and make the test output hard to read, so we ignore them. warnings.simplefilter("ignore") logging.getLogger().setLevel(logging.ERROR) From 882425ded5ae210c7092b87f4ea6bc871784ae89 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 31 May 2024 07:24:47 +0200 Subject: [PATCH 1033/1392] Add missing newline in `prec-commit-config.yaml` Just to be sure the coming linting won't be disturbed by that. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 02950db8c..fe966adad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,4 +25,4 @@ repos: - repo: https://github.com/abravalheri/validate-pyproject rev: v0.16 hooks: - - id: validate-pyproject \ No newline at end of file + - id: validate-pyproject From 59a0c88a08de4b35608d82b107844915a787f192 Mon Sep 17 00:00:00 2001 From: Andrej730 Date: Mon, 3 Jun 2024 00:26:11 +0500 Subject: [PATCH 1034/1392] Fix IndexFile items argument type Error before commit: path: os.PathLike = ... repo = git.Repo(path_dir) repo.index.add(path) --- git/index/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index b8161ea52..fc4474cac 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -658,7 +658,7 @@ def _to_relative_path(self, path: PathLike) -> PathLike: return os.path.relpath(path, self.repo.working_tree_dir) def _preprocess_add_items( - self, items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]] + self, items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]] ) -> Tuple[List[PathLike], List[BaseIndexEntry]]: """Split the items into two lists of path strings and BaseEntries.""" paths = [] @@ -749,7 +749,7 @@ def _entries_for_paths( def add( self, - items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]], + items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]], force: bool = True, fprogress: Callable = lambda *args: None, path_rewriter: Union[Callable[..., PathLike], None] = None, @@ -976,7 +976,7 @@ def _items_to_rela_paths( @default_index def remove( self, - items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]], + items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]], working_tree: bool = False, **kwargs: Any, ) -> List[str]: @@ -1036,7 +1036,7 @@ def remove( @default_index def move( self, - items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]], + items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]], skip_errors: bool = False, **kwargs: Any, ) -> List[Tuple[str, str]]: From 77fb5f06bd86a02f481a1d34ca0938bb5b7f5219 Mon Sep 17 00:00:00 2001 From: Andrej730 Date: Mon, 3 Jun 2024 00:28:45 +0500 Subject: [PATCH 1035/1392] Specify DiffIndex generic type Example before this commit: repo = git.Repo(path_dir) diff = repo.index.diff(None) modified_files = [d for d in repo.index.diff(None)] reveal_type(modified_files) # list[Unknown] instead of list[Diff] --- git/diff.py | 8 ++++---- git/index/base.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/git/diff.py b/git/diff.py index f89b12d98..e9f7e209f 100644 --- a/git/diff.py +++ b/git/diff.py @@ -187,7 +187,7 @@ def diff( paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, **kwargs: Any, - ) -> "DiffIndex": + ) -> "DiffIndex[Diff]": """Create diffs between two items being trees, trees and index or an index and the working tree. Detects renames automatically. @@ -581,7 +581,7 @@ def _pick_best_path(cls, path_match: bytes, rename_match: bytes, path_fallback_m return None @classmethod - def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoInterrupt"]) -> DiffIndex: + def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoInterrupt"]) -> DiffIndex["Diff"]: """Create a new :class:`DiffIndex` from the given process output which must be in patch format. @@ -674,7 +674,7 @@ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoIn return index @staticmethod - def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> None: + def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"]) -> None: lines = lines_bytes.decode(defenc) # Discard everything before the first colon, and the colon itself. @@ -747,7 +747,7 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non index.append(diff) @classmethod - def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex": + def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex[Diff]": """Create a new :class:`DiffIndex` from the given process output which must be in raw format. diff --git a/git/index/base.py b/git/index/base.py index fc4474cac..28b60a880 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1478,7 +1478,7 @@ def diff( paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, **kwargs: Any, - ) -> git_diff.DiffIndex: + ) -> git_diff.DiffIndex[git_diff.Diff]: """Diff this index against the working copy or a :class:`~git.objects.tree.Tree` or :class:`~git.objects.commit.Commit` object. From 491e134d2a930d12cc4250951e9e986dbab2be2d Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 4 Jun 2024 06:58:07 -0400 Subject: [PATCH 1036/1392] Fix Improper Import Order Breaking `fuzz_submodule` Fuzzer ClusterFuzz runs of the `fuzz_submodule` target have been failing because the `git` import was placed before the condition that sets the Git executable path. The order in which `git` is imported matters because it attempts to find a Git executable as the import is loaded (via `refresh()` in `git/__init__.py`.) As per #1909, we configure the ClusterFuzz environment to use a bundled Git executable via the env variable condition in all fuzz targets. --- fuzzing/fuzz-targets/fuzz_submodule.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index 92b569949..ca47690ea 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -4,12 +4,13 @@ import tempfile from configparser import ParsingError from utils import is_expected_exception_message, get_max_filename_length -from git import Repo, GitCommandError, InvalidGitRepositoryError if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # pragma: no cover path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary +from git import Repo, GitCommandError, InvalidGitRepositoryError + if not sys.warnoptions: # pragma: no cover # The warnings filter below can be overridden by passing the -W option # to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable. From d59708812f50362f526d4c6aa67b7218d12024f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Krzy=C5=9Bk=C3=B3w?= Date: Sat, 8 Jun 2024 01:23:21 +0200 Subject: [PATCH 1037/1392] Add deprecation test for DiffIndex.iter_change_type --- test/deprecation/test_basic.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/deprecation/test_basic.py b/test/deprecation/test_basic.py index 6235a836c..3bf0287c7 100644 --- a/test/deprecation/test_basic.py +++ b/test/deprecation/test_basic.py @@ -31,7 +31,7 @@ if TYPE_CHECKING: from pathlib import Path - from git.diff import Diff + from git.diff import Diff, DiffIndex from git.objects.commit import Commit # ------------------------------------------------------------------------ @@ -54,6 +54,12 @@ def diff(commit: "Commit") -> Generator["Diff", None, None]: yield diff +@pytest.fixture +def diffs(commit: "Commit") -> Generator["DiffIndex", None, None]: + """Fixture to supply a DiffIndex.""" + yield commit.diff(NULL_TREE) + + def test_diff_renamed_warns(diff: "Diff") -> None: """The deprecated Diff.renamed property issues a deprecation warning.""" with pytest.deprecated_call(): @@ -122,3 +128,10 @@ def test_iterable_obj_inheriting_does_not_warn() -> None: class Derived(IterableObj): pass + + +def test_diff_iter_change_type(diffs: "DiffIndex") -> None: + """The internal DiffIndex.iter_change_type function issues no deprecation warning.""" + with assert_no_deprecation_warning(): + for change_type in diffs.change_type: + [*diffs.iter_change_type(change_type=change_type)] From e1c660d224a1d27469c9275940c9db841570b8d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Krzy=C5=9Bk=C3=B3w?= <34622465+kamilkrzyskow@users.noreply.github.com> Date: Tue, 28 May 2024 05:53:44 +0200 Subject: [PATCH 1038/1392] Fix iter_change_type diff renamed property to prevent warning --- git/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/diff.py b/git/diff.py index e9f7e209f..8d2646f99 100644 --- a/git/diff.py +++ b/git/diff.py @@ -325,7 +325,7 @@ def iter_change_type(self, change_type: Lit_change_type) -> Iterator[T_Diff]: yield diffidx elif change_type == "C" and diffidx.copied_file: yield diffidx - elif change_type == "R" and diffidx.renamed: + elif change_type == "R" and diffidx.renamed_file: yield diffidx elif change_type == "M" and diffidx.a_blob and diffidx.b_blob and diffidx.a_blob != diffidx.b_blob: yield diffidx From d50b2e3245f472637c6b86722d6dd969fb4c7183 Mon Sep 17 00:00:00 2001 From: Almaz Ilaletdinov Date: Sun, 9 Jun 2024 14:00:05 +0300 Subject: [PATCH 1039/1392] Use contextlib.suppress instead of except: pass --- gitdb/db/loose.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 256fec9cf..87cde864e 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -2,6 +2,8 @@ # # This module is part of GitDB and is released under # the New BSD License: https://opensource.org/license/bsd-3-clause/ +from contextlib import suppress + from gitdb.db.base import ( FileDBBase, ObjectDBR, @@ -90,10 +92,8 @@ def readable_db_object_path(self, hexsha): """ :return: readable object path to the object identified by hexsha :raise BadObject: If the object file does not exist""" - try: + with suppress(KeyError): return self._hexsha_to_file[hexsha] - except KeyError: - pass # END ignore cache misses # try filesystem From f1ec1f15ec13e369bb5a4d758e94d7877e481ed3 Mon Sep 17 00:00:00 2001 From: Nick Papior Date: Thu, 13 Jun 2024 14:35:03 +0200 Subject: [PATCH 1040/1392] fixed doc to not faulty do #1924 Signed-off-by: Nick Papior --- test/test_docs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_docs.py b/test/test_docs.py index b3547c1de..cc0bbf26a 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -469,11 +469,11 @@ def test_references_and_objects(self, rw_dir): # ![30-test_references_and_objects] # [31-test_references_and_objects] - git = repo.git - git.checkout("HEAD", b="my_new_branch") # Create a new branch. - git.branch("another-new-one") - git.branch("-D", "another-new-one") # Pass strings for full control over argument order. - git.for_each_ref() # '-' becomes '_' when calling it. + git_cmd = repo.git + git_cmd.checkout("HEAD", b="my_new_branch") # Create a new branch. + git_cmd.branch("another-new-one") + git_cmd.branch("-D", "another-new-one") # Pass strings for full control over argument order. + git_cmd.for_each_ref() # '-' becomes '_' when calling it. # ![31-test_references_and_objects] repo.git.clear_cache() From d35998f5f420db780a30d73b703296498e3aa531 Mon Sep 17 00:00:00 2001 From: Guillaume Cardoen Date: Mon, 17 Jun 2024 09:28:32 +0200 Subject: [PATCH 1041/1392] fix: fix beginning whitespace error --- git/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/diff.py b/git/diff.py index 8d2646f99..9c6ae59e0 100644 --- a/git/diff.py +++ b/git/diff.py @@ -695,7 +695,7 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"] change_type: Lit_change_type = cast(Lit_change_type, _change_type[0]) score_str = "".join(_change_type[1:]) score = int(score_str) if score_str.isdigit() else None - path = path.strip() + path = path.strip("\n") a_path = path.encode(defenc) b_path = path.encode(defenc) deleted_file = False From 9910a886ddeb05a39b774e9f3520837fd9a76dca Mon Sep 17 00:00:00 2001 From: Guillaume Cardoen Date: Mon, 17 Jun 2024 09:31:51 +0200 Subject: [PATCH 1042/1392] test: add test for diff with beginning whitespace --- test/test_diff.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/test_diff.py b/test/test_diff.py index 928a9f428..6cae3fbf2 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -529,3 +529,23 @@ def test_diff_patch_with_external_engine(self, rw_dir): self.assertEqual(len(index_against_head), 1) index_against_working_tree = repo.index.diff(None, create_patch=True) self.assertEqual(len(index_against_working_tree), 1) + + @with_rw_directory + def test_beginning_space(self, rw_dir): + # Create a file beginning by a whitespace + repo = Repo.init(rw_dir) + file = osp.join(rw_dir, " file.txt") + with open(file, "w") as f: + f.write("hello world") + repo.git.add(Git.polish_url(file)) + repo.index.commit("first commit") + + # Diff the commit with an empty tree + # and check the paths + diff_index = repo.head.commit.diff(NULL_TREE) + d = diff_index[0] + a_path = d.a_path + b_path = d.b_path + self.assertEqual(a_path, " file.txt") + self.assertEqual(b_path, " file.txt") + \ No newline at end of file From 97fad9cb8322e510647cf58cc023702a7b7e077f Mon Sep 17 00:00:00 2001 From: Guillaume Cardoen Date: Tue, 18 Jun 2024 08:51:00 +0200 Subject: [PATCH 1043/1392] style: ruff --- test/test_diff.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test_diff.py b/test/test_diff.py index 6cae3fbf2..612fbd9e0 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -539,7 +539,7 @@ def test_beginning_space(self, rw_dir): f.write("hello world") repo.git.add(Git.polish_url(file)) repo.index.commit("first commit") - + # Diff the commit with an empty tree # and check the paths diff_index = repo.head.commit.diff(NULL_TREE) @@ -548,4 +548,3 @@ def test_beginning_space(self, rw_dir): b_path = d.b_path self.assertEqual(a_path, " file.txt") self.assertEqual(b_path, " file.txt") - \ No newline at end of file From f96eb0cdaeb6e33bf7725e1fb0385509f6030969 Mon Sep 17 00:00:00 2001 From: Patrick Massot Date: Mon, 24 Jun 2024 14:02:53 -0400 Subject: [PATCH 1044/1392] Change aliases to work around mypy issue. Fixes #1934 Note this should also gives better LSP support to these property aliases. --- git/remote.py | 11 +++++++++-- git/repo/base.py | 24 ++++++++++++++++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/git/remote.py b/git/remote.py index 37c991d27..15e360064 100644 --- a/git/remote.py +++ b/git/remote.py @@ -828,8 +828,15 @@ def remove(cls, repo: "Repo", name: str) -> str: name._clear_cache() return name - # `rm` is an alias. - rm = remove + @classmethod + def rm(cls, repo: "Repo", name: str) -> str: + """Alias of remove. + Remove the remote with the given name. + + :return: + The passed remote name to remove + """ + return cls.remove(repo, name) def rename(self, new_name: str) -> "Remote": """Rename self to the given `new_name`. diff --git a/git/repo/base.py b/git/repo/base.py index 51ea76901..346248ddb 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -402,6 +402,17 @@ def heads(self) -> "IterableList[Head]": """ return Head.list_items(self) + @property + def branches(self) -> "IterableList[Head]": + """Alias for heads. + A list of :class:`~git.refs.head.Head` objects representing the branch heads + in this repo. + + :return: + ``git.IterableList(Head, ...)`` + """ + return self.heads + @property def references(self) -> "IterableList[Reference]": """A list of :class:`~git.refs.reference.Reference` objects representing tags, @@ -412,11 +423,16 @@ def references(self) -> "IterableList[Reference]": """ return Reference.list_items(self) - # Alias for references. - refs = references + @property + def refs(self) -> "IterableList[Reference]": + """Alias for references. + A list of :class:`~git.refs.reference.Reference` objects representing tags, + heads and remote references. - # Alias for heads. - branches = heads + :return: + ``git.IterableList(Reference, ...)`` + """ + return self.references @property def index(self) -> "IndexFile": From 366a60760cea066b40ed33815fa8256b25afdfcc Mon Sep 17 00:00:00 2001 From: jirka Date: Tue, 16 Jul 2024 12:35:36 +0200 Subject: [PATCH 1045/1392] exclude: test/fixtures/ --- .pre-commit-config.yaml | 1 + test/fixtures/.gitconfig | 2 +- test/fixtures/blame | 2 +- test/fixtures/cat_file_blob | 2 +- test/fixtures/git_config | 1 + test/fixtures/git_config_with_empty_value | 2 +- test/fixtures/rev_list_bisect_all | 1 + test/fixtures/rev_list_commit_diffs | 1 + test/fixtures/rev_list_commit_idabbrev | 1 + test/fixtures/rev_list_commit_stats | 1 + 10 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 50f430084..5491c4297 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,6 +19,7 @@ repos: rev: v4.5.0 hooks: - id: end-of-file-fixer + exclude: test/fixtures/ - id: check-toml - id: check-yaml - id: check-merge-conflict diff --git a/test/fixtures/.gitconfig b/test/fixtures/.gitconfig index f6c25c15a..6a0459f6b 100644 --- a/test/fixtures/.gitconfig +++ b/test/fixtures/.gitconfig @@ -1,3 +1,3 @@ [alias] rbi = "!g() { git rebase -i origin/${1:-master} ; } ; g" - expush = "!f() { git branch -f tmp ; { git rbi $1 && git push ; } ; git reset --hard tmp ; git rebase origin/${1:-master}; } ; f" + expush = "!f() { git branch -f tmp ; { git rbi $1 && git push ; } ; git reset --hard tmp ; git rebase origin/${1:-master}; } ; f" \ No newline at end of file diff --git a/test/fixtures/blame b/test/fixtures/blame index 949976c5d..10c141dda 100644 --- a/test/fixtures/blame +++ b/test/fixtures/blame @@ -128,4 +128,4 @@ b6e1b765e0c15586a2c5b9832854f95defd71e1f 23 23 634396b2f541a9f2d58b00be1a07f0c358b999b3 11 24 2 VERSION = '1.0.0' 634396b2f541a9f2d58b00be1a07f0c358b999b3 12 25 - end + end \ No newline at end of file diff --git a/test/fixtures/cat_file_blob b/test/fixtures/cat_file_blob index 802992c42..70c379b63 100644 --- a/test/fixtures/cat_file_blob +++ b/test/fixtures/cat_file_blob @@ -1 +1 @@ -Hello world +Hello world \ No newline at end of file diff --git a/test/fixtures/git_config b/test/fixtures/git_config index d3066d86e..a8cad56e8 100644 --- a/test/fixtures/git_config +++ b/test/fixtures/git_config @@ -43,3 +43,4 @@ # inclusions should be processed immediately [sec] var1 = value1_main + diff --git a/test/fixtures/git_config_with_empty_value b/test/fixtures/git_config_with_empty_value index 83de84c8b..0427caea5 100644 --- a/test/fixtures/git_config_with_empty_value +++ b/test/fixtures/git_config_with_empty_value @@ -1,4 +1,4 @@ [color] ui [core] - filemode = true + filemode = true \ No newline at end of file diff --git a/test/fixtures/rev_list_bisect_all b/test/fixtures/rev_list_bisect_all index 60d382d01..342ea94ae 100644 --- a/test/fixtures/rev_list_bisect_all +++ b/test/fixtures/rev_list_bisect_all @@ -48,3 +48,4 @@ committer David Aguilar 1220418344 -0700 This resolves the issue mentioned in that thread. Signed-off-by: David Aguilar + diff --git a/test/fixtures/rev_list_commit_diffs b/test/fixtures/rev_list_commit_diffs index c39df2061..20397e2e4 100644 --- a/test/fixtures/rev_list_commit_diffs +++ b/test/fixtures/rev_list_commit_diffs @@ -5,3 +5,4 @@ author Tom Preston-Werner 1193200199 -0700 committer Tom Preston-Werner 1193200199 -0700 fix some initialization warnings + diff --git a/test/fixtures/rev_list_commit_idabbrev b/test/fixtures/rev_list_commit_idabbrev index 6266df93e..9385ba713 100644 --- a/test/fixtures/rev_list_commit_idabbrev +++ b/test/fixtures/rev_list_commit_idabbrev @@ -5,3 +5,4 @@ author tom 1195608462 -0800 committer tom 1195608462 -0800 fix tests on other machines + diff --git a/test/fixtures/rev_list_commit_stats b/test/fixtures/rev_list_commit_stats index c78aadeb5..60aa8cf58 100644 --- a/test/fixtures/rev_list_commit_stats +++ b/test/fixtures/rev_list_commit_stats @@ -4,3 +4,4 @@ author Tom Preston-Werner 1191997100 -0700 committer Tom Preston-Werner 1191997100 -0700 initial grit setup + From 1c88b0a734142cfc05114ad2ca0794c565294fb9 Mon Sep 17 00:00:00 2001 From: jirka Date: Tue, 7 May 2024 19:32:10 +0200 Subject: [PATCH 1046/1392] use codespell --- .pre-commit-config.yaml | 6 ++++++ pyproject.toml | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 551d8be34..23272bc25 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,10 @@ repos: +- repo: https://github.com/codespell-project/codespell + rev: v2.2.4 + hooks: + - id: codespell + additional_dependencies: [tomli] + - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.4.3 hooks: diff --git a/pyproject.toml b/pyproject.toml index ee54edb78..7fc809a6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,3 +78,8 @@ lint.unfixable = [ "test/**" = [ "B018", # useless-expression ] + +[tool.codespell] +#skip = '*.po,*.ts,./src/3rdParty,./src/Test' +#count = true +quiet-level = 3 \ No newline at end of file From 2ce013cc0043f7968f126ea38482a32077efa991 Mon Sep 17 00:00:00 2001 From: Jirka Date: Tue, 7 May 2024 19:38:44 +0200 Subject: [PATCH 1047/1392] fix & skip --- .pre-commit-config.yaml | 1 + git/index/base.py | 2 +- git/remote.py | 2 +- pyproject.toml | 3 ++- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 23272bc25..03730febd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,7 @@ repos: hooks: - id: codespell additional_dependencies: [tomli] + args: ["--write-changes"] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.4.3 diff --git a/git/index/base.py b/git/index/base.py index 28b60a880..a317e71c0 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1443,7 +1443,7 @@ def reset( key = entry_key(path, 0) self.entries[key] = nie[key] except KeyError: - # If key is not in theirs, it musn't be in ours. + # If key is not in theirs, it mustn't be in ours. try: del self.entries[key] except KeyError: diff --git a/git/remote.py b/git/remote.py index 15e360064..1e09e210e 100644 --- a/git/remote.py +++ b/git/remote.py @@ -250,7 +250,7 @@ def _from_line(cls, remote: "Remote", line: str) -> "PushInfo": flags |= cls.NEW_TAG elif "[new branch]" in summary: flags |= cls.NEW_HEAD - # uptodate encoded in control character + # up-to-date encoded in control character else: # Fast-forward or forced update - was encoded in control character, # but we parse the old and new commit. diff --git a/pyproject.toml b/pyproject.toml index 7fc809a6d..8b4522824 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ lint.unfixable = [ ] [tool.codespell] -#skip = '*.po,*.ts,./src/3rdParty,./src/Test' +skip = 'test/fixtures/reflog_*' +ignore-words-list="gud,doesnt" #count = true quiet-level = 3 \ No newline at end of file From 93993b201458fd18059e97fa25a08a14fae2af1f Mon Sep 17 00:00:00 2001 From: jirka Date: Wed, 17 Jul 2024 12:31:02 +0200 Subject: [PATCH 1048/1392] fixing --- .pre-commit-config.yaml | 3 ++- README.md | 2 +- doc/source/changes.rst | 2 +- git/objects/util.py | 6 +++--- pyproject.toml | 3 +-- test/test_exc.py | 2 +- test/test_index.py | 6 +++--- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03730febd..692c7fa2a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,8 @@ repos: hooks: - id: codespell additional_dependencies: [tomli] - args: ["--write-changes"] + # args: ["--write-changes"] # consider enabling for auto-fif + exclude: "test/fixtures/" - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.4.3 diff --git a/README.md b/README.md index d365a6584..59c6f995b 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ In the less common case that you do not want to install test dependencies, `pip #### With editable *dependencies* (not preferred, and rarely needed) -In rare cases, you may want to work on GitPython and one or both of its [gitdb](https://github.com/gitpython-developers/gitdb) and [smmap](https://github.com/gitpython-developers/smmap) dependencies at the same time, with changes in your local working copy of gitdb or smmap immediatley reflected in the behavior of your local working copy of GitPython. This can be done by making editable installations of those dependencies in the same virtual environment where you install GitPython. +In rare cases, you may want to work on GitPython and one or both of its [gitdb](https://github.com/gitpython-developers/gitdb) and [smmap](https://github.com/gitpython-developers/smmap) dependencies at the same time, with changes in your local working copy of gitdb or smmap immediately reflected in the behavior of your local working copy of GitPython. This can be done by making editable installations of those dependencies in the same virtual environment where you install GitPython. If you want to do that *and* you want the versions in GitPython's git submodules to be used, then pass `-e git/ext/gitdb` and/or `-e git/ext/gitdb/gitdb/ext/smmap` to `pip install`. This can be done in any order, and in separate `pip install` commands or the same one, so long as `-e` appears before *each* path. For example, you can install GitPython, gitdb, and smmap editably in the currently active virtual environment this way: diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 0bc757134..3c903423c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -20,7 +20,7 @@ https://github.com/gitpython-developers/GitPython/releases/tag/3.1.42 3.1.41 ====== -This release is relevant for security as it fixes a possible arbitary +This release is relevant for security as it fixes a possible arbitrary code execution on Windows. See this PR for details: https://github.com/gitpython-developers/GitPython/pull/1792 diff --git a/git/objects/util.py b/git/objects/util.py index 5c56e6134..a68d701f5 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -568,11 +568,11 @@ def addToStack( yield rval # Only continue to next level if this is appropriate! - nd = d + 1 - if depth > -1 and nd > depth: + next_d = d + 1 + if depth > -1 and next_d > depth: continue - addToStack(stack, item, branch_first, nd) + addToStack(stack, item, branch_first, next_d) # END for each item on work stack diff --git a/pyproject.toml b/pyproject.toml index 8b4522824..603e2597c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,6 @@ lint.unfixable = [ ] [tool.codespell] -skip = 'test/fixtures/reflog_*' ignore-words-list="gud,doesnt" #count = true -quiet-level = 3 \ No newline at end of file +quiet-level = 3 diff --git a/test/test_exc.py b/test/test_exc.py index c1eae7240..2e979f5a1 100644 --- a/test/test_exc.py +++ b/test/test_exc.py @@ -52,7 +52,7 @@ _streams_n_substrings = ( None, - "steram", + "stream", "ομορφο stream", ) diff --git a/test/test_index.py b/test/test_index.py index b92258c92..2684cfd81 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1018,7 +1018,7 @@ class Mocked: @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Absent, reason="Can't run a hook on Windows without bash.exe.", - rasies=HookExecutionError, + raises=HookExecutionError, ) @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, @@ -1077,7 +1077,7 @@ def test_hook_uses_shell_not_from_cwd(self, rw_dir, case): @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Absent, reason="Can't run a hook on Windows without bash.exe.", - rasies=HookExecutionError, + raises=HookExecutionError, ) @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, @@ -1120,7 +1120,7 @@ def test_pre_commit_hook_fail(self, rw_repo): @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Absent, reason="Can't run a hook on Windows without bash.exe.", - rasies=HookExecutionError, + raises=HookExecutionError, ) @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Wsl, From 813520c123d44a8cf87a219fc710faeb1f1559ca Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:34:14 +0200 Subject: [PATCH 1049/1392] Apply suggestions from code review --- git/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/remote.py b/git/remote.py index 1e09e210e..9de3dace4 100644 --- a/git/remote.py +++ b/git/remote.py @@ -250,7 +250,7 @@ def _from_line(cls, remote: "Remote", line: str) -> "PushInfo": flags |= cls.NEW_TAG elif "[new branch]" in summary: flags |= cls.NEW_HEAD - # up-to-date encoded in control character + # `uptodate` encoded in control character else: # Fast-forward or forced update - was encoded in control character, # but we parse the old and new commit. From ce8a69a4141d2149bac2cbf56ea7d4b1f2ed7257 Mon Sep 17 00:00:00 2001 From: Jonas Scharpf Date: Wed, 17 Jul 2024 11:01:09 +0200 Subject: [PATCH 1050/1392] Add type of change to files_dict of a commit This allows to not only get the total, inserted or deleted number of lines being changed but also the type of change like Added (A), Copied (C), Deleted (D), Modified (M), Renamed (R), type changed (T), Unmerged (U), Unknown (X), or pairing Broken (B) --- AUTHORS | 1 + git/objects/commit.py | 24 +++++++++++++++++------- git/types.py | 1 + git/util.py | 4 +++- test/fixtures/diff_numstat | 5 +++-- test/test_commit.py | 9 ++++++--- test/test_stats.py | 12 +++++++++--- 7 files changed, 40 insertions(+), 16 deletions(-) diff --git a/AUTHORS b/AUTHORS index 9311b3962..45b14c961 100644 --- a/AUTHORS +++ b/AUTHORS @@ -54,5 +54,6 @@ Contributors are: -Wenhan Zhu -Eliah Kagan -Ethan Lin +-Jonas Scharpf Portions derived from other open source works and are clearly marked. diff --git a/git/objects/commit.py b/git/objects/commit.py index d957c9051..0ceb46609 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -377,15 +377,25 @@ def stats(self) -> Stats: :return: :class:`Stats` """ - if not self.parents: - text = self.repo.git.diff_tree(self.hexsha, "--", numstat=True, no_renames=True, root=True) - text2 = "" - for line in text.splitlines()[1:]: + + def process_lines(lines: List[str]) -> str: + text = "" + for file_info, line in zip(lines, lines[len(lines) // 2 :]): + change_type = file_info.split("\t")[0][-1] (insertions, deletions, filename) = line.split("\t") - text2 += "%s\t%s\t%s\n" % (insertions, deletions, filename) - text = text2 + text += "%s\t%s\t%s\t%s\n" % (change_type, insertions, deletions, filename) + return text + + if not self.parents: + lines = self.repo.git.diff_tree( + self.hexsha, "--", numstat=True, no_renames=True, root=True, raw=True + ).splitlines()[1:] + text = process_lines(lines) else: - text = self.repo.git.diff(self.parents[0].hexsha, self.hexsha, "--", numstat=True, no_renames=True) + lines = self.repo.git.diff( + self.parents[0].hexsha, self.hexsha, "--", numstat=True, no_renames=True, raw=True + ).splitlines() + text = process_lines(lines) return Stats._list_from_string(self.repo, text) @property diff --git a/git/types.py b/git/types.py index 584450146..cce184530 100644 --- a/git/types.py +++ b/git/types.py @@ -248,6 +248,7 @@ class Files_TD(TypedDict): insertions: int deletions: int lines: int + change_type: str class Total_TD(TypedDict): diff --git a/git/util.py b/git/util.py index 11f963e02..9e8ac821d 100644 --- a/git/util.py +++ b/git/util.py @@ -910,6 +910,7 @@ class Stats: deletions = number of deleted lines as int insertions = number of inserted lines as int lines = total number of lines changed as int, or deletions + insertions + change_type = type of change as str, A|C|D|M|R|T|U|X|B ``full-stat-dict`` @@ -938,7 +939,7 @@ def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": "files": {}, } for line in text.splitlines(): - (raw_insertions, raw_deletions, filename) = line.split("\t") + (change_type, raw_insertions, raw_deletions, filename) = line.split("\t") insertions = raw_insertions != "-" and int(raw_insertions) or 0 deletions = raw_deletions != "-" and int(raw_deletions) or 0 hsh["total"]["insertions"] += insertions @@ -949,6 +950,7 @@ def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": "insertions": insertions, "deletions": deletions, "lines": insertions + deletions, + "change_type": change_type, } hsh["files"][filename.strip()] = files_dict return Stats(hsh["total"], hsh["files"]) diff --git a/test/fixtures/diff_numstat b/test/fixtures/diff_numstat index 44c6ca2d5..b76e467eb 100644 --- a/test/fixtures/diff_numstat +++ b/test/fixtures/diff_numstat @@ -1,2 +1,3 @@ -29 18 a.txt -0 5 b.txt +M 29 18 a.txt +M 0 5 b.txt +A 7 0 c.txt \ No newline at end of file diff --git a/test/test_commit.py b/test/test_commit.py index 5832258de..37c66e3e7 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -135,9 +135,12 @@ def test_stats(self): commit = self.rorepo.commit("33ebe7acec14b25c5f84f35a664803fcab2f7781") stats = commit.stats - def check_entries(d): + def check_entries(d, has_change_type=False): assert isinstance(d, dict) - for key in ("insertions", "deletions", "lines"): + keys = ("insertions", "deletions", "lines") + if has_change_type: + keys += ("change_type",) + for key in keys: assert key in d # END assertion helper @@ -148,7 +151,7 @@ def check_entries(d): assert "files" in stats.total for _filepath, d in stats.files.items(): - check_entries(d) + check_entries(d, True) # END for each stated file # Check that data is parsed properly. diff --git a/test/test_stats.py b/test/test_stats.py index eec73c802..91d2cf6ae 100644 --- a/test/test_stats.py +++ b/test/test_stats.py @@ -14,13 +14,19 @@ def test_list_from_string(self): output = fixture("diff_numstat").decode(defenc) stats = Stats._list_from_string(self.rorepo, output) - self.assertEqual(2, stats.total["files"]) - self.assertEqual(52, stats.total["lines"]) - self.assertEqual(29, stats.total["insertions"]) + self.assertEqual(3, stats.total["files"]) + self.assertEqual(59, stats.total["lines"]) + self.assertEqual(36, stats.total["insertions"]) self.assertEqual(23, stats.total["deletions"]) self.assertEqual(29, stats.files["a.txt"]["insertions"]) self.assertEqual(18, stats.files["a.txt"]["deletions"]) + self.assertEqual("M", stats.files["a.txt"]["change_type"]) self.assertEqual(0, stats.files["b.txt"]["insertions"]) self.assertEqual(5, stats.files["b.txt"]["deletions"]) + self.assertEqual("M", stats.files["b.txt"]["change_type"]) + + self.assertEqual(7, stats.files["c.txt"]["insertions"]) + self.assertEqual(0, stats.files["c.txt"]["deletions"]) + self.assertEqual("A", stats.files["c.txt"]["change_type"]) From 58a9a58f58e6aae220efda8ce95bf4c2e0fd9ca0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 24 Jul 2024 02:10:57 -0400 Subject: [PATCH 1051/1392] Use Alpine Linux in WSL on CI Some of the CI tests use WSL. This switches the WSL distribution from Debian to Alpine, which might be slightly faster. For the way it is being used here, the main expected speed improvement would be to how long the image would take to download, as Alpine is smaller. (The reason for this is thus unrelated to the reason for the Alpine docker CI test job added in #1826. There, the goal was to test on a wider variety of systems and environments, and that runs the whole test suite in Alpine. This just changes the WSL distro, used by a few tests on Windows, from Debian to Alpine.) Two things have changed that, taken together, have unblocked this: - https://github.com/Vampire/setup-wsl/issues/50 was fixed, so the action we are using is able to install Alpine Linux. See: https://github.com/gitpython-developers/GitPython/pull/1917#pullrequestreview-2081550232 - #1893 was fixed in #1888. So if switching the WSL distro from Debian to Alpine breaks any tests, including by making them fail in an unexpected way that raises the wrong exception, we are likely to find out. --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 031b0e6b2..61ab2206c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -46,7 +46,7 @@ jobs: if: startsWith(matrix.os, 'windows') uses: Vampire/setup-wsl@v3.1.1 with: - distribution: Debian + distribution: Alpine - name: Prepare this repo for tests run: | From ce5eefd90b6c652083ce615583b5ee62b39ae187 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 24 Jul 2024 02:39:03 -0400 Subject: [PATCH 1052/1392] Enable Python 3.8 and 3.9 on M1 runners These were excluded in 9ad28c3 (#1817) due to https://github.com/actions/setup-python/issues/808, which was later fixed by https://github.com/actions/python-versions/pull/259. Because Python 3.7 has been end-of-life for a while, it is very unlikely to have AArch64 builds added in python-versions for use on GitHub Actions CI runners (preinstalled or via setup-python). --- .github/workflows/pythonpackage.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 031b0e6b2..f3c837742 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -18,10 +18,6 @@ jobs: exclude: - os: "macos-14" python-version: "3.7" - - os: "macos-14" - python-version: "3.8" - - os: "macos-14" - python-version: "3.9" include: - experimental: false From 055394a548d19dded2ad9791a208bbcc54879b14 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 24 Jul 2024 03:25:31 -0400 Subject: [PATCH 1053/1392] Install bash in WSL Alpine distro Because Alpine Linux does not ship with bash, and the tests that use WSL use it. --- .github/workflows/pythonpackage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 61ab2206c..1902ecb19 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -47,6 +47,7 @@ jobs: uses: Vampire/setup-wsl@v3.1.1 with: distribution: Alpine + additional-packages: bash - name: Prepare this repo for tests run: | From c2bbaf47e14dac5f0470938b3ecf67836ca1695d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 24 Jul 2024 04:34:00 -0400 Subject: [PATCH 1054/1392] Remove the non-ARM macOS CI jobs This keeps only the macos-14 jobs, which run on Apple Silicon M1, and removes the macos-13 jobs, which ran on x86-64. Other operating systems jobs continue to run on x86-64 machines (and none on ARM, yet). Only the macOS jobs are removed. This change leaves Python 3.7 without any macOS test job. That is probably okay, since it has been end-of-life for some time, and it remains tested on Ubuntu and Windows. --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 49f6c5254..7547aecf9 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "macos-13", "macos-14", "windows-latest"] + os: ["ubuntu-latest", "macos-14", "windows-latest"] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] exclude: - os: "macos-14" From be6744b6e4365fc42996a1be7f026f133f992928 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 24 Jul 2024 04:38:06 -0400 Subject: [PATCH 1055/1392] Use the macos-latest label rather than macos-14 Currently they are the same. The macos-latest label will move to later versions automatically in the future, like the ubuntu-latest and windows-latest labels that we are already using. In this repo, the macos-14 label had been used originally because it was added before the migration of macos-latest to be macos-14 was completed. See https://github.com/github/roadmap/issues/926. It was kept for clarity of constrast with the macos-13 jobs that were also in use, some for the same Python versions. Now that the macos-13 jobs have been removed in c2bbaf4, the macos-latest label can be used here without confusion. --- .github/workflows/pythonpackage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 7547aecf9..0f1d17544 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -13,10 +13,10 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "macos-14", "windows-latest"] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] exclude: - - os: "macos-14" + - os: "macos-latest" python-version: "3.7" include: - experimental: false From af0cd933e84b9f83210c0f12f95a456606ee79e9 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 6 Jun 2024 02:17:25 -0400 Subject: [PATCH 1056/1392] Fix "OSError: [Errno 36] File name too long" in fuzz_submodule Fixes a bug in the `fuzz_submodule` harness where the fuzzed data can produce file names that exceed the maximum size allowed byt the OS. This issue came up previously and was fixed in #1922, but the submodule file name fixed here was missed in that PR. Fixes: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=69456 --- fuzzing/fuzz-targets/fuzz_submodule.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index ca47690ea..9f5828d8d 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -35,12 +35,13 @@ def TestOneInput(data): sub_repo = Repo.init(submodule_temp_dir, bare=fdp.ConsumeBool()) sub_repo.index.commit(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))) - submodule_name = f"submodule_{fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))}" + submodule_name = fdp.ConsumeUnicodeNoSurrogates( + fdp.ConsumeIntInRange(1, max(1, get_max_filename_length(repo.working_tree_dir))) + ) submodule_path = os.path.join(repo.working_tree_dir, submodule_name) - submodule_url = sub_repo.git_dir - submodule = repo.create_submodule(submodule_name, submodule_path, url=submodule_url) - repo.index.commit(f"Added submodule {submodule_name}") + submodule = repo.create_submodule(submodule_name, submodule_path, url=sub_repo.git_dir) + repo.index.commit("Added submodule") with submodule.config_writer() as writer: key_length = fdp.ConsumeIntInRange(1, max(1, fdp.remaining_bytes())) From 7de1556d3895c718f0f0772530ff7cde5457d9d8 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 8 Aug 2024 16:54:37 -0400 Subject: [PATCH 1057/1392] Filter out non-bug exceptions using a pre-defined exception list. This reduces false positive test failures by identifying and gracefully handling exceptions that are explicitly raised by GitPython, thus reducing the false-positive fuzzing test failure rate. --- fuzzing/fuzz-targets/fuzz_submodule.py | 56 +++++++++++++++---- fuzzing/oss-fuzz-scripts/build.sh | 2 +- .../container-environment-bootstrap.sh | 11 ++++ 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index 9f5828d8d..05c543bf8 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -1,16 +1,51 @@ +# ruff: noqa: E402 import atheris import sys import os +import traceback import tempfile from configparser import ParsingError -from utils import is_expected_exception_message, get_max_filename_length +from utils import get_max_filename_length +import re + +bundle_dir = os.path.dirname(os.path.abspath(__file__)) if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # pragma: no cover - path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) - os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + bundled_git_binary_path = os.path.join(bundle_dir, "git") + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = bundled_git_binary_path from git import Repo, GitCommandError, InvalidGitRepositoryError + +def load_exception_list(file_path): + """Load and parse the exception list from a file.""" + try: + with open(file_path, "r") as file: + lines = file.readlines() + exception_list = set() + for line in lines: + match = re.match(r"(.+):(\d+):", line) + if match: + file_path = match.group(1).strip() + line_number = int(match.group(2).strip()) + exception_list.add((file_path, line_number)) + return exception_list + except FileNotFoundError: + print("File not found: %s", file_path) + return set() + except Exception as e: + print("Error loading exception list: %s", e) + return set() + + +def check_exception_against_list(exception_list, exc_traceback): + """Check if the exception traceback matches any entry in the exception list.""" + for filename, lineno, _, _ in traceback.extract_tb(exc_traceback): + if (filename, lineno) in exception_list: + return True + return False + + if not sys.warnoptions: # pragma: no cover # The warnings filter below can be overridden by passing the -W option # to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable. @@ -89,17 +124,14 @@ def TestOneInput(data): BrokenPipeError, ): return -1 - except ValueError as e: - expected_messages = [ - "SHA is empty", - "Reference at", - "embedded null byte", - "This submodule instance does not exist anymore", - "cmd stdin was empty", - ] - if is_expected_exception_message(e, expected_messages): + except Exception as e: + exc_traceback = e.__traceback__ + exception_list = load_exception_list(os.path.join(bundle_dir, "explicit-exceptions-list.txt")) + if check_exception_against_list(exception_list, exc_traceback): + print("Exception matches an entry in the exception list.") return -1 else: + print("Exception does not match any entry in the exception list.") raise e diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh index e0b3a50ab..c156e872d 100644 --- a/fuzzing/oss-fuzz-scripts/build.sh +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -15,5 +15,5 @@ find "$SRC" -maxdepth 1 \ # Build fuzzers in $OUT. find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do - compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):." + compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):." --add-data="$SRC/explicit-exceptions-list.txt:." done diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh index bbdcf5357..af1ddf014 100755 --- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -91,6 +91,17 @@ create_seed_corpora_zips "$WORK/qa-assets/gitpython/corpora" prepare_dictionaries_for_fuzz_targets "$WORK/qa-assets/gitpython/dictionaries" "$SRC/gitpython/fuzzing" +pushd "$SRC/gitpython/" +# Search for 'raise' and 'assert' statements in Python files within GitPython's 'git/' directory and its submodules, +# remove trailing colons, and save to 'explicit-exceptions-list.txt'. This file can then be used by fuzz harnesses to +# check exception tracebacks: +# If an exception found by the fuzzer originated in a file + line number in explicit-exceptions-list.txt, then it is not a bug. + +git grep -n --recurse-submodules -e '\braise\b' -e '\bassert\b' -- "git/**/*.py" > "$SRC/explicit-exceptions-list.txt" + +popd + + # The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below. python3 -m pip install --upgrade pip # Upgrade to the latest versions known to work at the time the below changes were introduced: From 799b9cae745f50f2c0c590e8b3e19bfea199c463 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 8 Aug 2024 18:58:28 -0400 Subject: [PATCH 1058/1392] Improve `check_exception_against_list` matching logic using regex Changes: - `match_exception_with_traceback` uses regular expressions for more flexible matching of file paths and line numbers. This allows for partial matches and more complex patterns. - Improve `check_exception_against_list` by delegating to `match_exception_with_traceback` for checking tracebacks against exception list entries. - `load_exception_list`: Remains largely unchanged, as it correctly parses the file and line number from each exception entry. However, we ensure the set consists of regex patterns to match against tracebacks. --- fuzzing/fuzz-targets/fuzz_submodule.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index 05c543bf8..37f069079 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -31,21 +31,27 @@ def load_exception_list(file_path): exception_list.add((file_path, line_number)) return exception_list except FileNotFoundError: - print("File not found: %s", file_path) + print(f"File not found: {file_path}") return set() except Exception as e: - print("Error loading exception list: %s", e) + print(f"Error loading exception list: {e}") return set() -def check_exception_against_list(exception_list, exc_traceback): - """Check if the exception traceback matches any entry in the exception list.""" +def match_exception_with_traceback(exception_list, exc_traceback): + """Match exception traceback with the entries in the exception list.""" for filename, lineno, _, _ in traceback.extract_tb(exc_traceback): - if (filename, lineno) in exception_list: - return True + for file_pattern, line_pattern in exception_list: + if re.fullmatch(file_pattern, filename) and re.fullmatch(line_pattern, str(lineno)): + return True return False +def check_exception_against_list(exception_list, exc_traceback): + """Check if the exception traceback matches any entry in the exception list.""" + return match_exception_with_traceback(exception_list, exc_traceback) + + if not sys.warnoptions: # pragma: no cover # The warnings filter below can be overridden by passing the -W option # to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable. @@ -128,10 +134,8 @@ def TestOneInput(data): exc_traceback = e.__traceback__ exception_list = load_exception_list(os.path.join(bundle_dir, "explicit-exceptions-list.txt")) if check_exception_against_list(exception_list, exc_traceback): - print("Exception matches an entry in the exception list.") return -1 else: - print("Exception does not match any entry in the exception list.") raise e From 2e9c23995b70372a18edc4d0b143b6b522d3fb39 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 8 Aug 2024 19:38:06 -0400 Subject: [PATCH 1059/1392] Extract environment setup and exception checking boilerplate logic Changes: - Simplify exception handling in test harnesses via `handle_exception(e)` in the `except Exception as e:` block. - `setup_git_environment` is a step towards centralizing environment variable and logging configuration set up consistently across different fuzzing scripts. **Only applying it to a single test for now is an intentional choice in case it fails to work in the ClusterFuzz environment!** If it proves successful, a follow-up change set will be welcome. --- fuzzing/fuzz-targets/fuzz_submodule.py | 70 +++------------------ fuzzing/fuzz-targets/utils.py | 87 +++++++++++++++++++++++++- 2 files changed, 95 insertions(+), 62 deletions(-) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index 37f069079..634572bf2 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -1,67 +1,17 @@ -# ruff: noqa: E402 import atheris import sys import os -import traceback import tempfile from configparser import ParsingError -from utils import get_max_filename_length -import re - -bundle_dir = os.path.dirname(os.path.abspath(__file__)) - -if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # pragma: no cover - bundled_git_binary_path = os.path.join(bundle_dir, "git") - os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = bundled_git_binary_path - from git import Repo, GitCommandError, InvalidGitRepositoryError +from utils import ( + setup_git_environment, + handle_exception, + get_max_filename_length, +) - -def load_exception_list(file_path): - """Load and parse the exception list from a file.""" - try: - with open(file_path, "r") as file: - lines = file.readlines() - exception_list = set() - for line in lines: - match = re.match(r"(.+):(\d+):", line) - if match: - file_path = match.group(1).strip() - line_number = int(match.group(2).strip()) - exception_list.add((file_path, line_number)) - return exception_list - except FileNotFoundError: - print(f"File not found: {file_path}") - return set() - except Exception as e: - print(f"Error loading exception list: {e}") - return set() - - -def match_exception_with_traceback(exception_list, exc_traceback): - """Match exception traceback with the entries in the exception list.""" - for filename, lineno, _, _ in traceback.extract_tb(exc_traceback): - for file_pattern, line_pattern in exception_list: - if re.fullmatch(file_pattern, filename) and re.fullmatch(line_pattern, str(lineno)): - return True - return False - - -def check_exception_against_list(exception_list, exc_traceback): - """Check if the exception traceback matches any entry in the exception list.""" - return match_exception_with_traceback(exception_list, exc_traceback) - - -if not sys.warnoptions: # pragma: no cover - # The warnings filter below can be overridden by passing the -W option - # to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable. - import warnings - import logging - - # Fuzzing data causes some modules to generate a large number of warnings - # which are not usually interesting and make the test output hard to read, so we ignore them. - warnings.simplefilter("ignore") - logging.getLogger().setLevel(logging.ERROR) +# Setup the git environment +setup_git_environment() def TestOneInput(data): @@ -131,12 +81,10 @@ def TestOneInput(data): ): return -1 except Exception as e: - exc_traceback = e.__traceback__ - exception_list = load_exception_list(os.path.join(bundle_dir, "explicit-exceptions-list.txt")) - if check_exception_against_list(exception_list, exc_traceback): + if isinstance(e, ValueError) and "embedded null byte" in str(e): return -1 else: - raise e + return handle_exception(e) def main(): diff --git a/fuzzing/fuzz-targets/utils.py b/fuzzing/fuzz-targets/utils.py index f522d2959..97e6eab98 100644 --- a/fuzzing/fuzz-targets/utils.py +++ b/fuzzing/fuzz-targets/utils.py @@ -1,6 +1,9 @@ import atheris # pragma: no cover import os # pragma: no cover -from typing import List # pragma: no cover +import re # pragma: no cover +import traceback # pragma: no cover +import sys # pragma: no cover +from typing import Set, Tuple, List # pragma: no cover @atheris.instrument_func @@ -35,3 +38,85 @@ def get_max_filename_length(path: str) -> int: # pragma: no cover int: The maximum filename length. """ return os.pathconf(path, "PC_NAME_MAX") + + +@atheris.instrument_func +def read_lines_from_file(file_path: str) -> list: + """Read lines from a file and return them as a list.""" + try: + with open(file_path, "r") as f: + return [line.strip() for line in f if line.strip()] + except FileNotFoundError: + print(f"File not found: {file_path}") + return [] + except IOError as e: + print(f"Error reading file {file_path}: {e}") + return [] + + +@atheris.instrument_func +def load_exception_list(file_path: str = "explicit-exceptions-list.txt") -> Set[Tuple[str, str]]: + """Load and parse the exception list from a default or specified file.""" + try: + bundle_dir = os.path.dirname(os.path.abspath(__file__)) + full_path = os.path.join(bundle_dir, file_path) + lines = read_lines_from_file(full_path) + exception_list: Set[Tuple[str, str]] = set() + for line in lines: + match = re.match(r"(.+):(\d+):", line) + if match: + file_path: str = match.group(1).strip() + line_number: str = str(match.group(2).strip()) + exception_list.add((file_path, line_number)) + return exception_list + except Exception as e: + print(f"Error loading exception list: {e}") + return set() + + +@atheris.instrument_func +def match_exception_with_traceback(exception_list: Set[Tuple[str, str]], exc_traceback) -> bool: + """Match exception traceback with the entries in the exception list.""" + for filename, lineno, _, _ in traceback.extract_tb(exc_traceback): + for file_pattern, line_pattern in exception_list: + # Ensure filename and line_number are strings for regex matching + if re.fullmatch(file_pattern, filename) and re.fullmatch(line_pattern, str(lineno)): + return True + return False + + +@atheris.instrument_func +def check_exception_against_list(exc_traceback, exception_file: str = "explicit-exceptions-list.txt") -> bool: + """Check if the exception traceback matches any entry in the exception list.""" + exception_list = load_exception_list(exception_file) + return match_exception_with_traceback(exception_list, exc_traceback) + + +@atheris.instrument_func +def handle_exception(e: Exception) -> int: + """Encapsulate exception handling logic for reusability.""" + exc_traceback = e.__traceback__ + if check_exception_against_list(exc_traceback): + return -1 + else: + raise e + + +@atheris.instrument_func +def setup_git_environment() -> None: + """Set up the environment variables for Git.""" + bundle_dir = os.path.dirname(os.path.abspath(__file__)) + if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # pragma: no cover + bundled_git_binary_path = os.path.join(bundle_dir, "git") + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = bundled_git_binary_path + + if not sys.warnoptions: # pragma: no cover + # The warnings filter below can be overridden by passing the -W option + # to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable. + import warnings + import logging + + # Fuzzing data causes some modules to generate a large number of warnings + # which are not usually interesting and make the test output hard to read, so we ignore them. + warnings.simplefilter("ignore") + logging.getLogger().setLevel(logging.ERROR) From 27de8676c64b549038b4fdd994a20f1ce996ad5e Mon Sep 17 00:00:00 2001 From: David Lakin Date: Thu, 8 Aug 2024 20:35:13 -0400 Subject: [PATCH 1060/1392] Fix buggy `git grep` pathspec args To ensure that all necessary files are included in the explicit-exceptions-list.txt file and unwanted files and directories are not. --- .../container-environment-bootstrap.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh index af1ddf014..924a3cbf3 100755 --- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -92,12 +92,12 @@ create_seed_corpora_zips "$WORK/qa-assets/gitpython/corpora" prepare_dictionaries_for_fuzz_targets "$WORK/qa-assets/gitpython/dictionaries" "$SRC/gitpython/fuzzing" pushd "$SRC/gitpython/" -# Search for 'raise' and 'assert' statements in Python files within GitPython's 'git/' directory and its submodules, -# remove trailing colons, and save to 'explicit-exceptions-list.txt'. This file can then be used by fuzz harnesses to -# check exception tracebacks: -# If an exception found by the fuzzer originated in a file + line number in explicit-exceptions-list.txt, then it is not a bug. +# Search for 'raise' and 'assert' statements in Python files within GitPython's source code and submodules, saving the +# matched file path, line number, and line content to a file named 'explicit-exceptions-list.txt'. +# This file can then be used by fuzz harnesses to check exception tracebacks and filter out explicitly raised or otherwise +# anticipated exceptions to reduce false positive test failures. -git grep -n --recurse-submodules -e '\braise\b' -e '\bassert\b' -- "git/**/*.py" > "$SRC/explicit-exceptions-list.txt" +git grep -n --recurse-submodules -e '\braise\b' -e '\bassert\b' -- '*.py' -- ':!setup.py' -- ':!test/**' -- ':!fuzzing/**' > "$SRC/explicit-exceptions-list.txt" popd From 2ed33345667706c5755708e88c989ede06f2414f Mon Sep 17 00:00:00 2001 From: David Lakin Date: Fri, 9 Aug 2024 00:06:44 -0400 Subject: [PATCH 1061/1392] Fix order of environment setup and git module import The environment setup must happen before the `git` module is imported, otherwise GitPython won't be able to find the Git executable and raise an exception that causes the ClusterFuzz fuzzer runs to fail. --- fuzzing/fuzz-targets/fuzz_submodule.py | 2 +- pyproject.toml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index 634572bf2..997133b70 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -3,7 +3,6 @@ import os import tempfile from configparser import ParsingError -from git import Repo, GitCommandError, InvalidGitRepositoryError from utils import ( setup_git_environment, handle_exception, @@ -12,6 +11,7 @@ # Setup the git environment setup_git_environment() +from git import Repo, GitCommandError, InvalidGitRepositoryError def TestOneInput(data): diff --git a/pyproject.toml b/pyproject.toml index 603e2597c..6cf4b3f5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,10 @@ lint.unfixable = [ "test/**" = [ "B018", # useless-expression ] +"fuzzing/fuzz-targets/**" = [ + "E402", # environment setup must happen before the `git` module is imported, thus cannot happen at top of file +] + [tool.codespell] ignore-words-list="gud,doesnt" From 096851b61fa99df233176b090146efb52e524f48 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Fri, 9 Aug 2024 11:01:34 -0400 Subject: [PATCH 1062/1392] Gracefully handle `PermissionError` exceptions that crash fuzzer Fuzzing inputs sometimes produce directory paths that are protected inside the fuzzer execution environment. This is not an issue in GitPython's code, so it should not crash the fuzzer. Fixes OSS-Fuzz Issue 69456: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=69870 --- fuzzing/fuzz-targets/fuzz_submodule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index 997133b70..c2bf1e4fe 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -78,6 +78,7 @@ def TestOneInput(data): IsADirectoryError, NotADirectoryError, BrokenPipeError, + PermissionError, ): return -1 except Exception as e: From 7126ce16a03e0aea5ef4d031c62596992a6d7cb5 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 13 Aug 2024 01:09:37 -0400 Subject: [PATCH 1063/1392] Fuzzing: Gracefully Handle Uninteresting Error to Fix OSS-Fuzz Issue 71095 Fuzzing data can generate filenames that trigger: > OSError: [Errno 36] File name too long The changes here add handling for these exceptions because they di not indicate a bug and should not crash the fuzzer. a --- fuzzing/fuzz-targets/fuzz_submodule.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index c2bf1e4fe..d22b0aa5b 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -84,6 +84,8 @@ def TestOneInput(data): except Exception as e: if isinstance(e, ValueError) and "embedded null byte" in str(e): return -1 + elif isinstance(e, OSError) and "File name too long" in str(e): + return -1 else: return handle_exception(e) From d1582d181bfeb5138d9cae306b40dfa2fe87fe39 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 15 Aug 2024 18:08:22 -0400 Subject: [PATCH 1064/1392] Update versions of pre-commit hooks --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 692c7fa2a..7d93876ed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/codespell-project/codespell - rev: v2.2.4 + rev: v2.3.0 hooks: - id: codespell additional_dependencies: [tomli] @@ -8,7 +8,7 @@ repos: exclude: "test/fixtures/" - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.3 + rev: v0.6.0 hooks: - id: ruff args: ["--fix"] @@ -17,14 +17,14 @@ repos: exclude: ^git/ext/ - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.6 + rev: v0.10.0.1 hooks: - id: shellcheck args: [--color] exclude: ^test/fixtures/polyglot$|^git/ext/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: end-of-file-fixer exclude: test/fixtures/ @@ -33,6 +33,6 @@ repos: - id: check-merge-conflict - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.16 + rev: v0.19 hooks: - id: validate-pyproject From 016fa44a64ac244de2335b00338af67e3f8585ee Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 15 Aug 2024 18:09:06 -0400 Subject: [PATCH 1065/1392] Have codespell ignore words that cause new false positives --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6cf4b3f5d..090972eed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,6 @@ lint.unfixable = [ [tool.codespell] -ignore-words-list="gud,doesnt" +ignore-words-list="afile,assertIn,doesnt,gud,uptodate" #count = true quiet-level = 3 From c82bb65fd263603b374b925f61483efc47c2a264 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 15 Aug 2024 18:13:28 -0400 Subject: [PATCH 1066/1392] Fix a spelling error that codespell didn't catch --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7d93876ed..90b899f8e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: codespell additional_dependencies: [tomli] - # args: ["--write-changes"] # consider enabling for auto-fif + # args: ["--write-changes"] # consider enabling for auto-fix exclude: "test/fixtures/" - repo: https://github.com/astral-sh/ruff-pre-commit From 9556f63a965877db19002849d7bfeec71e84a2c7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 15 Aug 2024 18:19:03 -0400 Subject: [PATCH 1067/1392] Drop suggestion to auto-fix spelling (many false positives) --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 90b899f8e..c47d9a2c7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,6 @@ repos: hooks: - id: codespell additional_dependencies: [tomli] - # args: ["--write-changes"] # consider enabling for auto-fix exclude: "test/fixtures/" - repo: https://github.com/astral-sh/ruff-pre-commit From 7a138eea78fd922b21f6049d273aaeca5f02bfb0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 15 Aug 2024 18:55:48 -0400 Subject: [PATCH 1068/1392] Fix small inconsistencies in test/fixtures/ exclusions --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c47d9a2c7..f5635b2a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: codespell additional_dependencies: [tomli] - exclude: "test/fixtures/" + exclude: ^test/fixtures/ - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.6.0 @@ -26,7 +26,7 @@ repos: rev: v4.6.0 hooks: - id: end-of-file-fixer - exclude: test/fixtures/ + exclude: ^test/fixtures/ - id: check-toml - id: check-yaml - id: check-merge-conflict From 53ec790e0dbc1ec9e4451394edb5c572c807b817 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 15 Aug 2024 19:01:02 -0400 Subject: [PATCH 1069/1392] Fix inconsistent indentation --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f5635b2a0..0cbf5aa73 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,4 +34,4 @@ repos: - repo: https://github.com/abravalheri/validate-pyproject rev: v0.19 hooks: - - id: validate-pyproject + - id: validate-pyproject From bdfa280f6dd412464419dd133ad02781cd27a312 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 15 Aug 2024 19:02:14 -0400 Subject: [PATCH 1070/1392] Temporarily let end-of-file-fixer break LICENSE-BSD symlink On Windows, when `core.symlinks` is `false` or unset (since it defaults to `false` on Windows), Git checks out symbolic links as regular files whose contents are symlinks' target paths. Modifying those regular files and committing the changes alters the symlink target in the repository, and when they are checked out as actual symlinks, the targets are different. But the `end-of-file-fixer` pre-commit hook automatically adds newlines to the end of regular files that lack them. It doesn't do this on actual symlinks, but it does do it to regular files that stand in for symlinks. This causes it to carry a risk of breaking symlinks if it is run on Windows and the changes committed, and it is easy to miss that this will happen because `git diff` output shows it the same way as other additions of absent newlines. This deliberately commits the change that end-of-file-fixer makes to the `LICENSE-BSD` symlink, in order to allow a mitigation beyond just excluding that symlink (or replacing it with a regular file) to be tested. This change must be undone, of course. --- fuzzing/LICENSE-BSD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzing/LICENSE-BSD b/fuzzing/LICENSE-BSD index ea5b60640..4f88f81bf 120000 --- a/fuzzing/LICENSE-BSD +++ b/fuzzing/LICENSE-BSD @@ -1 +1 @@ -../LICENSE \ No newline at end of file +../LICENSE From 965ea8bebcd768f6cadbc6cae6b7fe65868f1fb6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 15 Aug 2024 20:17:24 -0400 Subject: [PATCH 1071/1392] Enable check-symlinks pre-commit hook Rationale: - Small but likely benefit in general, since there are no currently foreseen intentional use cases of committing of broken/dangling symlinks in this project. So such symlinks that arise are likely unintentional. - If the end-of-file-fixer hook has run on a Windows system where `core.symlinks` has *not* been set to `true`, and symlinks' paths have not been excluded, then a newline character is added to the end of the path held in the regular file Git checks out to stand in for the symlink. Because it is not actually a symlink, this will not detect the problem at that time (regardless of the order in which this and that hook run relative to each other). But when it is then run on CI on a system where symlinks are checked out, it will detect the problem. --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0cbf5aa73..3f6892687 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,7 @@ repos: hooks: - id: end-of-file-fixer exclude: ^test/fixtures/ + - id: check-symlinks - id: check-toml - id: check-yaml - id: check-merge-conflict From e9782487b8119147aa0c456c708f61ca7e3139e1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 15 Aug 2024 20:24:36 -0400 Subject: [PATCH 1072/1392] Revert "Temporarily let end-of-file-fixer break LICENSE-BSD symlink" This reverts commit bdfa280f6dd412464419dd133ad02781cd27a312. --- fuzzing/LICENSE-BSD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzing/LICENSE-BSD b/fuzzing/LICENSE-BSD index 4f88f81bf..ea5b60640 120000 --- a/fuzzing/LICENSE-BSD +++ b/fuzzing/LICENSE-BSD @@ -1 +1 @@ -../LICENSE +../LICENSE \ No newline at end of file From cae0d8743a31fb0eda3c224a45c14de8cabd0d90 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 15 Aug 2024 20:28:46 -0400 Subject: [PATCH 1073/1392] Don't fix end-of-file in files named like licenses The unanchored `LICENSE` and `COPYING` alternatives match the pattern anywhere, and therefore exclude the currently used path `fuzzing/LICENSE-BSD`. License files are more likely than other files in this project to be introduced as symlinks, and less likely to be noticed immediately if they break. Symlinks can be checked out as regular files when `core.symlinks` is set to `false`, which is rare outside of Windows but is the default behavior when unset on Windows. This exclusion fixes the current problem that end-of-file-fixer breaks those links by adding a newline character to the end (the symlinks are checked out broken if that is committed). It also guards against most future cases involving licenses, though possibly not all, and not other unrelated cases where symlinks may be used for other purposes. Although the pre-commit-hooks repository also provides a destroyed-symlinks hook that detects the situation of a symlink that has been replaced by a regular file, this does not add that hook, because this situation is not inherently a problem. The code here does not require symlinks to be checked out to work, and adding that would break significant uses of the repository on Windows. Note that this leaves the situation where a license file may be a symlink to another license file and may thus be checked out as a regular file containing that file's path. However, it is easy to understand that situation and manually follow the path. That differs from the scenario where a symlink is created but broken, because attempting to open it gives an error, and the error message is often non-obvious, reporting that a file is not found but giving the name of the symlink rather than its target. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f6892687..424cc5f37 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: rev: v4.6.0 hooks: - id: end-of-file-fixer - exclude: ^test/fixtures/ + exclude: ^test/fixtures/|COPYING|LICENSE - id: check-symlinks - id: check-toml - id: check-yaml From 16fc99fee45412c3dae44bdd7f59d921a11c00b3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 17 Aug 2024 12:51:13 -0400 Subject: [PATCH 1074/1392] Upgrade sphinx to ~7.1.2 The old pinned version and its explicitly constrained dependencies are retained for now for Python 3.7 so that documentation can be built even with 3.7. (This could maybe be removed soon as a preliminary step toward evenutally dropping 3.7 support.) For Python 3.8 and higher, version 7.1.2 is used, allowing later patch versions but constrained to remain 7.1.*. This is so the same versions are likely to be selected on all Python version from 3.8 and higher, to minimize small differences in generated documentation that different versions could give, and also to simplify debugging. The reason to upgrade Sphinx now is to suppport Python 3.13, which shall be (and, in the pre-releases available, is) incompatible with versions of Sphinx below 6.2. This is because those earlier Sphinx versions use the deprecated `imghdr` module, which 3.13 removes: - https://docs.python.org/3.13/whatsnew/3.13.html#whatsnew313-pep594 - https://github.com/python/cpython/issues/104818 On old versions of Sphinx, that gives the error: Extension error: Could not import extension sphinx.builders.epub3 (exception: No module named 'imghdr') Using Sphinx 6.2 is sufficient to avoid this, but there do not seem to be any disadvantages for GitPython to remain below 7.1.2. The reason we did not upgrade Sphinx before is that, last time we considered doing so, we ran into a problem of new warnings (that we treat as errors). This is detailed in the "Can we upgrade Sphinx?" section of #1802, especially the "What Sphinx 5 is talking about" subsection. The problem is warnings about `Actor` when it comes in through type annotations: WARNING: more than one target found for cross-reference 'Actor': git.objects.util.Actor, git.util.Actor So this includes other changes to fix that problem as well. The solution used here is to import `Actor` even when `TYPE_CHECKING` is `false`, and write it unquoted in annotations, as `Actor` rather than `"Actor"`. This allows Sphinx to discern where it should consider it to be located, for the purpose of linking to its documentation. This does not incur overhead, because: - The affected modules already have imports from `git.util`, so also importing `Actor` from `git.util` does not cause any modules to be imported that were not imported otherwise, nor even to be imported at a different time. - Even if that that had not been the case, most modules in GitPython including `git.util` have members imported them into the top-level `git` module in `git.__init__` when `git` is imported (and thus when any Python submodule of `git` is imported). The only disadvantage is the presence of the additional name in those modules at runtime, which a user might inadvertently use and thus write brittle code that could break if it is later removed. But: - The affected modules define `__all__` and do not include `"Actor"` in `__all__`, so it is non-public. - There are many places in GitPython (and most Python libraries) where the onus is already on the author of code that uses the library to avoid doing this. The reasons for this approach, rather than any of several others, were: 1. I did not write out the annotations as `git.util.Actor` to resolve the ambiguity because annotations should, and for some uses must, also be interpretable as expressions. But while `from git.util import Actor` works and makes `Actor` available, `git.util.Actor` cannot be used as an expression even after `import git.util`. This is because, even after such an import, `git.util` actually refers to `git.index.util`. This is as detailed in the warnings issued when it is accessed, originally from an overly broad `*` import but retained because removing it could be a breaking change. See `git/__init__.py` for details. 2. The reason I did not write out the annotations as `git.objects.util.Actor` to resolve the ambiguity is that not all occurrences where Sphinx needed to be told which module to document it as being from were within the `git.objects` Python submodule. Two of the warnings were in `git/objects/tag.py`, where annotating it that way would not be confusing. But the other two were in `git/index/base.py`. 3. Although removing `Actor` from `git.objects.util.__all__` would resolve the ambiguity, this should not be done, because: - This would be a breaking change. - It seems to be there deliberately, since `git.objects.util` contains other members that relate to it directly. 4. The reasons I did not take this opportunity to move the contents of `git/util.py` to a new file in that directory and make `git/util.py` re-export the contents, even though this would allow a solution analogous to (1) but for the new module to be used, while also simplifying importing elsewhere, were: - That seems like a change that should be done separately, based either on the primary benefit to users or on a greater need for it. - If and when that is done, it may make sense to change the interface as well. For example, `git/util.py` has a number of members that it makes available for use throughout GitPython but that are deliberately omitted from `__all__` and are meant as non-public outside the project. One approach would be to make a module with a leading `_` for these "internal" members, and another public ones with everything else. But that also cannot be decided based on the considerations that motivate the changes here. - The treatment of `HIDE_WINDOWS_KNOWN_ERRORS`, which is public in `git/util.py` and which currently *does* have an effect, will need to be considered. Although it cannot be re-bound by assigning to `git.util.HIDE_WINDOWS_KNOWN_ERRORS` because the `git.util` subexpression would evaluate to `git.index.util`, there may be code that re-binds it in another way, such as by accessing the module through `sys.modules`. Unlike functions and classes that should not be monkey-patched from code outside GitPython's test suite anyway, this attribute may reasonably be assigned to, so it matters what module it is actually in, unless the action of assigning to it elsewhere is customized dynamically to carry over to the "real" place. 5. An alternative solution that may be reasonable in the near future is to modify `reference.rst` so duplicate documentation is no longer emitted for functions and classes that are defined in one place but imported and "re-exported" elsewhere. I suspect this may solve the problem, allowing the `Actor` imports to go back under `if TYPE_CHECKING:` and to annotate with `"Actor"` again while still running `make -C doc html` with no warnings. However, this would entail design decisions about how to still document those members. They should probably have links to where they are fully documented. So an entry for `Actor` in the section of `reference.rst` for `git.objects.util` would still exist, but be very short and refer to the full autodoc item for `Actor` the section for `git.util`. Since a `:class:` reference to `git.objects.util.Actor` should still go to the stub that links to `git.util.Actor`, it is not obvious that solving the duplication in documentation generated for `reference.rst` ought to be done in a way that would address the ambiguity Sphinx warns about, even if it *can* be done in such a way. Therefore, that should also be a separate consideration and, if warranted, a separate change. --- doc/requirements.txt | 13 +++++++------ git/index/base.py | 6 +++--- git/objects/tag.py | 5 ++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 7769af5ae..a90a7a496 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,8 +1,9 @@ -sphinx == 4.3.2 +sphinx >= 7.1.2, < 7.2 ; python_version >= "3.8" +sphinx == 4.3.2 ; python_version < "3.8" +sphinxcontrib-applehelp >= 1.0.2, <= 1.0.4 ; python_version < "3.8" +sphinxcontrib-devhelp == 1.0.2 ; python_version < "3.8" +sphinxcontrib-htmlhelp >= 2.0.0, <= 2.0.1 ; python_version < "3.8" +sphinxcontrib-qthelp == 1.0.3 ; python_version < "3.8" +sphinxcontrib-serializinghtml == 1.1.5 ; python_version < "3.8" sphinx_rtd_theme -sphinxcontrib-applehelp >= 1.0.2, <= 1.0.4 -sphinxcontrib-devhelp == 1.0.2 -sphinxcontrib-htmlhelp >= 2.0.0, <= 2.0.1 -sphinxcontrib-qthelp == 1.0.3 -sphinxcontrib-serializinghtml == 1.1.5 sphinx-autodoc-typehints diff --git a/git/index/base.py b/git/index/base.py index a317e71c0..47925ad1c 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -28,6 +28,7 @@ from git.objects import Blob, Commit, Object, Submodule, Tree from git.objects.util import Serializable from git.util import ( + Actor, LazyMixin, LockedFD, join_path_native, @@ -76,7 +77,6 @@ from git.refs.reference import Reference from git.repo import Repo - from git.util import Actor Treeish = Union[Tree, Commit, str, bytes] @@ -1117,8 +1117,8 @@ def commit( message: str, parent_commits: Union[List[Commit], None] = None, head: bool = True, - author: Union[None, "Actor"] = None, - committer: Union[None, "Actor"] = None, + author: Union[None, Actor] = None, + committer: Union[None, Actor] = None, author_date: Union[datetime.datetime, str, None] = None, commit_date: Union[datetime.datetime, str, None] = None, skip_hooks: bool = False, diff --git a/git/objects/tag.py b/git/objects/tag.py index a3bb0b882..88671d316 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -14,7 +14,7 @@ import sys from git.compat import defenc -from git.util import hex_to_bin +from git.util import Actor, hex_to_bin from . import base from .util import get_object_type_by_name, parse_actor_and_date @@ -30,7 +30,6 @@ if TYPE_CHECKING: from git.repo import Repo - from git.util import Actor from .blob import Blob from .commit import Commit @@ -64,7 +63,7 @@ def __init__( binsha: bytes, object: Union[None, base.Object] = None, tag: Union[None, str] = None, - tagger: Union[None, "Actor"] = None, + tagger: Union[None, Actor] = None, tagged_date: Union[int, None] = None, tagger_tz_offset: Union[int, None] = None, message: Union[str, None] = None, From 44f7a738b85f75d86516a3ee1f128f4098fbcda6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 18 Aug 2024 14:02:40 -0400 Subject: [PATCH 1075/1392] Don't support building documentation on Python 3.7 This removes the specially cased alternative lower versions of `sphinx` and its dependencies that, since #1954, were only for Python 3.7. As discussed in comments there, this simplifies the documentation dependencies and avoids a situation where the version of Python used to build the documentation has a noticeable effect on the generated result. This also conditions running the "Documentation" step in the main CI test workflow (`pythonpackage.yml`) on the Python version not being 3.7 (otherwise the job would always fail). The only change this makes to the support status of GitPython on Python 3.7 is to no longer support building documentation on 3.7. GitPython can still be installed and used on 3.7 (though usually this would not be a good idea, outside of testing, since Python 3.7 itself has not been supported by the Python Software Foundation for quite some time). In addition, the documentation, which can be built on any version >= 3.8 (including 3.13 starting in #1954) is no less relevant to usage on Python 3.7 than it was before. --- .github/workflows/pythonpackage.yml | 1 + doc/requirements.txt | 8 +------- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 0f1d17544..292c9fc86 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -99,6 +99,7 @@ jobs: continue-on-error: false - name: Documentation + if: matrix.python-version != '3.7' run: | pip install ".[doc]" make -C doc html diff --git a/doc/requirements.txt b/doc/requirements.txt index a90a7a496..81140d898 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,9 +1,3 @@ -sphinx >= 7.1.2, < 7.2 ; python_version >= "3.8" -sphinx == 4.3.2 ; python_version < "3.8" -sphinxcontrib-applehelp >= 1.0.2, <= 1.0.4 ; python_version < "3.8" -sphinxcontrib-devhelp == 1.0.2 ; python_version < "3.8" -sphinxcontrib-htmlhelp >= 2.0.0, <= 2.0.1 ; python_version < "3.8" -sphinxcontrib-qthelp == 1.0.3 ; python_version < "3.8" -sphinxcontrib-serializinghtml == 1.1.5 ; python_version < "3.8" +sphinx >= 7.1.2, < 7.2 sphinx_rtd_theme sphinx-autodoc-typehints From f2254af5d3fd183ec150740d517bd0f8070fc67d Mon Sep 17 00:00:00 2001 From: Andrej730 Date: Sat, 14 Sep 2024 10:45:53 +0500 Subject: [PATCH 1076/1392] _to_relative_path to support mixing slashes and backslashes Working on Windows you sometime end up having some paths with backslashes (windows native) and some with slashes - this PR will resolve the issue using gitpython for those kind of cases (see example below). It will also fix the issues if paths contain redundant separators or "..". ``` import git repo = git.Repo(r"C:\gittest") repo.index.add(r"C:\gittest\1.txt") # Traceback (most recent call last): # File "c:\second_test.py", line 5, in # repo.index.add(r"C:/gittest/2.txt") # File "Python311\Lib\site-packages\git\index\base.py", line 879, in add # paths, entries = self._preprocess_add_items(items) # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # File "Python311\Lib\site-packages\git\index\base.py", line 672, in _preprocess_add_items # paths.append(self._to_relative_path(item)) # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # File "Python311\Lib\site-packages\git\index\base.py", line 657, in _to_relative_path # raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir)) # ValueError: Absolute path 'C:/gittest/2.txt' is not in git repository at 'C:\\gittest' repo.index.add(r"C:/gittest/2.txt") repo.index.commit("test") ``` --- git/index/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/index/base.py b/git/index/base.py index 47925ad1c..7f53e614a 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -653,7 +653,7 @@ def _to_relative_path(self, path: PathLike) -> PathLike: return path if self.repo.bare: raise InvalidGitRepositoryError("require non-bare repository") - if not str(path).startswith(str(self.repo.working_tree_dir)): + if not osp.normpath(str(path)).startswith(osp.normpath(str(self.repo.working_tree_dir))): raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir)) return os.path.relpath(path, self.repo.working_tree_dir) From ca06b11efde845080354dac71e9062ea6d63ab84 Mon Sep 17 00:00:00 2001 From: Andrej730 Date: Sat, 14 Sep 2024 16:51:41 +0500 Subject: [PATCH 1077/1392] test adding a file using non-normalized path --- test/test_index.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/test_index.py b/test/test_index.py index 2684cfd81..efd5b83a6 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1181,6 +1181,18 @@ def test_index_add_pathlike(self, rw_repo): rw_repo.index.add(file) + @with_rw_repo("HEAD") + def test_index_add_non_normalized_path(self, rw_repo): + git_dir = Path(rw_repo.git_dir) + + file = git_dir / "file.txt" + file.touch() + non_normalized_path = file.as_posix() + if os.name != "nt": + non_normalized_path = non_normalized_path.replace("/", "\\") + + rw_repo.index.add(non_normalized_path) + class TestIndexUtils: @pytest.mark.parametrize("file_path_type", [str, Path]) From 46740590f7918fd5b789c95db7e41fbda06fb46f Mon Sep 17 00:00:00 2001 From: Andrej730 Date: Sat, 14 Sep 2024 16:52:56 +0500 Subject: [PATCH 1078/1392] Remove redundant path normalization for working_tree_dir --- git/index/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/index/base.py b/git/index/base.py index 7f53e614a..39cc9143c 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -653,7 +653,7 @@ def _to_relative_path(self, path: PathLike) -> PathLike: return path if self.repo.bare: raise InvalidGitRepositoryError("require non-bare repository") - if not osp.normpath(str(path)).startswith(osp.normpath(str(self.repo.working_tree_dir))): + if not osp.normpath(str(path)).startswith(str(self.repo.working_tree_dir)): raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir)) return os.path.relpath(path, self.repo.working_tree_dir) From 8327b82a1079f667006f649cb3f1bbdcc8792955 Mon Sep 17 00:00:00 2001 From: Andrej730 Date: Sat, 14 Sep 2024 21:18:18 +0500 Subject: [PATCH 1079/1392] Fix test failing on unix --- test/test_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_index.py b/test/test_index.py index efd5b83a6..c586a0b5a 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1189,7 +1189,7 @@ def test_index_add_non_normalized_path(self, rw_repo): file.touch() non_normalized_path = file.as_posix() if os.name != "nt": - non_normalized_path = non_normalized_path.replace("/", "\\") + non_normalized_path = "/" + non_normalized_path[1:].replace("/", "//") rw_repo.index.add(non_normalized_path) From 49ca9099dc75d0d686ec6737da36637cbee1c000 Mon Sep 17 00:00:00 2001 From: No big deal <69958306+alex20230721@users.noreply.github.com> Date: Sat, 5 Oct 2024 17:23:19 +0800 Subject: [PATCH 1080/1392] Update base.py (#1965) Improve documentation around opening repositories. Co-authored-by: Sebastian Thiel --- git/repo/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/repo/base.py b/git/repo/base.py index 346248ddb..db89cdf41 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -179,7 +179,7 @@ def __init__( R"""Create a new :class:`Repo` instance. :param path: - The path to either the root git directory or the bare git repo:: + The path to either the worktree directory or the .git directory itself:: repo = Repo("/Users/mtrier/Development/git-python") repo = Repo("/Users/mtrier/Development/git-python.git") From 5bc95043792c2412b05263fb4bfca67d7923645c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez-Mondrag=C3=B3n?= Date: Wed, 9 Oct 2024 01:01:55 -0600 Subject: [PATCH 1081/1392] Add support for Python 3.13 --- .github/workflows/pythonpackage.yml | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 73b3902c3..8e0ff8e5c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] include: - experimental: false continue-on-error: ${{ matrix.experimental }} diff --git a/setup.py b/setup.py index f67f7a58f..51065c960 100755 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3 :: Only", ] ) From b38cbc43354523ffcd59a58c5a3aded054bd4442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez-Mondrag=C3=B3n?= Date: Wed, 9 Oct 2024 01:05:50 -0600 Subject: [PATCH 1082/1392] Use older ubuntu to get Python 3.7 --- .github/workflows/pythonpackage.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 8e0ff8e5c..40e1c0ae0 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -8,12 +8,17 @@ on: [push, pull_request, workflow_dispatch] jobs: build: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + os: [ubuntu-latest] + experimental: [false] include: - - experimental: false + - python-version: "3.7" + os: ubuntu-22.04 + experimental: false continue-on-error: ${{ matrix.experimental }} steps: From 1bb465122f9673c9834b094c49d815148e84b8eb Mon Sep 17 00:00:00 2001 From: Florent Valette Date: Mon, 14 Oct 2024 21:39:37 +0200 Subject: [PATCH 1083/1392] git,remote: use universal new lines for fetch/pull stderr capture See https://github.com/gitpython-developers/GitPython/issues/1969 stderr parser call RemoteProgress update on each line received. With universal_newlines set to False, there is a mixup between line feed and carriage return. In the `handle_process_output` thread, this is thus seen as a single line for the whole output on each steps. --- git/remote.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/remote.py b/git/remote.py index 9de3dace4..20e42b412 100644 --- a/git/remote.py +++ b/git/remote.py @@ -894,7 +894,7 @@ def _get_fetch_info_from_stderr( None, progress_handler, finalizer=None, - decode_streams=True, + decode_streams=False, kill_after_timeout=kill_after_timeout, ) @@ -1071,7 +1071,7 @@ def fetch( Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_fetch_options) proc = self.repo.git.fetch( - "--", self, *args, as_process=True, with_stdout=False, universal_newlines=False, v=verbose, **kwargs + "--", self, *args, as_process=True, with_stdout=False, universal_newlines=True, v=verbose, **kwargs ) res = self._get_fetch_info_from_stderr(proc, progress, kill_after_timeout=kill_after_timeout) if hasattr(self.repo.odb, "update_cache"): @@ -1125,7 +1125,7 @@ def pull( Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_pull_options) proc = self.repo.git.pull( - "--", self, refspec, with_stdout=False, as_process=True, universal_newlines=False, v=True, **kwargs + "--", self, refspec, with_stdout=False, as_process=True, universal_newlines=True, v=True, **kwargs ) res = self._get_fetch_info_from_stderr(proc, progress, kill_after_timeout=kill_after_timeout) if hasattr(self.repo.odb, "update_cache"): From 52cceaf2663422a79a0f1d21f905eb132e46b556 Mon Sep 17 00:00:00 2001 From: Florent Valette Date: Tue, 15 Oct 2024 18:04:44 +0200 Subject: [PATCH 1084/1392] git,cmd: add encoding arg to popen if universal newlines is True --- git/cmd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/git/cmd.py b/git/cmd.py index 90fc39cd6..2048a43fa 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1269,6 +1269,7 @@ def execute( stdout=stdout_sink, shell=shell, universal_newlines=universal_newlines, + encoding=defenc if universal_newlines else None, **subprocess_kwargs, ) except cmd_not_found_exception as err: From d6cdb67bcaa2cf606bfc0a9295aacb54677ea86d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 15 Oct 2024 20:35:29 +0200 Subject: [PATCH 1085/1392] See if python 3.7 still works when using an older Ubuntu version. This should be undone once python 3.7 is EOL. --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 292c9fc86..747db62f0 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: ["ubuntu-22.04", "macos-latest", "windows-latest"] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] exclude: - os: "macos-latest" From 74a0eabbc03209593ea1562498802359ae8a3db7 Mon Sep 17 00:00:00 2001 From: Jonathan Dekhtiar Date: Mon, 23 Dec 2024 23:02:58 -0500 Subject: [PATCH 1086/1392] Potential Race Condition Fix - OS Rename & Chmod --- gitdb/db/loose.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 87cde864e..ccefe4006 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -54,6 +54,7 @@ import tempfile import os import sys +import time __all__ = ('LooseObjectDB', ) @@ -205,7 +206,7 @@ def store(self, istream): # END assure target stream is closed except: if tmp_path: - os.remove(tmp_path) + remove(tmp_path) raise # END assure tmpfile removal on error @@ -228,9 +229,25 @@ def store(self, istream): rename(tmp_path, obj_path) # end rename only if needed - # make sure its readable for all ! It started out as rw-- tmp file - # but needs to be rwrr - chmod(obj_path, self.new_objects_mode) + # Ensure rename is actually done and file is stable + # Retry up to 14 times - exponential wait & retry in ms. + # The total maximum wait time is 1000ms, which should be vastly enough for the + # OS to return and commit the file to disk. + for exp_backoff_ms in [1, 4, 9, 16, 25, 36, 49, 64, 81, 100, 121, 144, 169, 181]: + with suppress(PermissionError): + # make sure its readable for all ! It started out as rw-- tmp file + # but needs to be rwrr + chmod(obj_path, self.new_objects_mode) + break + time.sleep(exp_backoff_ms / 1000.0) + else: + raise PermissionError( + "Impossible to apply `chmod` to file {}".format(obj_path) + ) + + # Cleanup + with suppress(FileNotFoundError): + remove(tmp_path) # END handle dry_run istream.binsha = hex_to_bin(hexsha) From b71e2730c3dcab148816f0193a45550ef0a38c79 Mon Sep 17 00:00:00 2001 From: Jonathan DEKHTIAR Date: Sun, 29 Dec 2024 19:44:26 -0500 Subject: [PATCH 1087/1392] Update gitdb/db/loose.py Co-authored-by: Sebastian Thiel --- gitdb/db/loose.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index ccefe4006..03d387e86 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -245,9 +245,6 @@ def store(self, istream): "Impossible to apply `chmod` to file {}".format(obj_path) ) - # Cleanup - with suppress(FileNotFoundError): - remove(tmp_path) # END handle dry_run istream.binsha = hex_to_bin(hexsha) From f31bfa378c8840d38d31e7e11ef2b84f191a491e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 2 Jan 2025 08:04:16 +0100 Subject: [PATCH 1088/1392] bump patch level prior to new release --- doc/source/changes.rst | 6 ++++++ smmap/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 947c71a81..dc243eb0c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +****** +v5.0.2 +****** + +- remove a usage of mktemp + ****** v5.0.1 ****** diff --git a/smmap/__init__.py b/smmap/__init__.py index 429f47aed..657c8c5af 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (5, 0, 1) +version_info = (5, 0, 2) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From 104138c742a56d85bd2cb2cd8a9f90336daa5483 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 2 Jan 2025 08:15:19 +0100 Subject: [PATCH 1089/1392] bump patch level to prepare for next release --- doc/source/changes.rst | 6 ++++++ gitdb/__init__.py | 2 +- gitdb/ext/smmap | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 0b8de13d5..b4340e4dc 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ######### +****** +4.0.12 +****** + +- various improvements - please see the release on GitHub for details. + ****** 4.0.11 ****** diff --git a/gitdb/__init__.py b/gitdb/__init__.py index 9b77e9f2d..1fb7df893 100644 --- a/gitdb/__init__.py +++ b/gitdb/__init__.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 11) +version_info = (4, 0, 12) __version__ = '.'.join(str(i) for i in version_info) # default imports diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 04dd2103e..f31bfa378 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 04dd2103ee6e0b7483889e5feda25053c6df2b52 +Subproject commit f31bfa378c8840d38d31e7e11ef2b84f191a491e diff --git a/setup.py b/setup.py index 51065c960..3a9154311 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/gitdb" -version_info = (4, 0, 11) +version_info = (4, 0, 12) __version__ = '.'.join(str(i) for i in version_info) setup( From 775cfe8299ea5474f605935469359a9d1cdb49dc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 2 Jan 2025 08:20:58 +0100 Subject: [PATCH 1090/1392] update scripts to allow release (copied from smmap) --- Makefile | 42 +++++++----------------------------------- build-release.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 35 deletions(-) create mode 100755 build-release.sh diff --git a/Makefile b/Makefile index a0a2d0e01..20436bbc4 100644 --- a/Makefile +++ b/Makefile @@ -1,40 +1,12 @@ -PYTHON = python3 -SETUP = $(PYTHON) setup.py -TESTFLAGS = +.PHONY: all clean release force_release -all:: +all: @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all -release:: clean - # Check if latest tag is the current head we're releasing - echo "Latest tag = $$(git tag | sort -nr | head -n1)" - echo "HEAD SHA = $$(git rev-parse head)" - echo "Latest tag SHA = $$(git tag | sort -nr | head -n1 | xargs git rev-parse)" - @test "$$(git rev-parse head)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" - make force_release +clean: + rm -rf build/ dist/ .eggs/ .tox/ -force_release:: clean - git push --tags - python3 -m build --sdist --wheel +force_release: clean + ./build-release.sh twine upload dist/* - -doc:: - make -C doc/ html - -build:: - $(SETUP) build - $(SETUP) build_ext -i - -build_ext:: - $(SETUP) build_ext -i - -install:: - $(SETUP) install - -clean:: - $(SETUP) clean --all - rm -f *.so - -coverage:: build - PYTHONPATH=. $(PYTHON) -m pytest --cov=gitdb gitdb - + git push --tags origin master diff --git a/build-release.sh b/build-release.sh new file mode 100755 index 000000000..5840e4472 --- /dev/null +++ b/build-release.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# This script builds a release. If run in a venv, it auto-installs its tools. +# You may want to run "make release" instead of running this script directly. + +set -eEu + +function release_with() { + $1 -m build --sdist --wheel +} + +if test -n "${VIRTUAL_ENV:-}"; then + deps=(build twine) # Install twine along with build, as we need it later. + echo "Virtual environment detected. Adding packages: ${deps[*]}" + pip install --quiet --upgrade "${deps[@]}" + echo 'Starting the build.' + release_with python +else + function suggest_venv() { + venv_cmd='python -m venv env && source env/bin/activate' + printf "HELP: To avoid this error, use a virtual-env with '%s' instead.\n" "$venv_cmd" + } + trap suggest_venv ERR # This keeps the original exit (error) code. + echo 'Starting the build.' + release_with python3 # Outside a venv, use python3. +fi From e51bf80ad576256f2fbeead41ea3f0b667c77055 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 2 Jan 2025 08:24:01 +0100 Subject: [PATCH 1091/1392] update GitDB submodule to latest pubslished version --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 3d3e9572d..775cfe829 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 3d3e9572dc452fea53d328c101b3d1440bbefe40 +Subproject commit 775cfe8299ea5474f605935469359a9d1cdb49dc From fb1b05124f1070ed56231a782daee0ffce9e1372 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 2 Jan 2025 08:27:54 +0100 Subject: [PATCH 1092/1392] bump patch level to prepare new version --- VERSION | 2 +- doc/source/changes.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d1bf6638d..e6af1c454 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.43 +3.1.44 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 3c903423c..00a3c660e 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ========= +3.1.44 +====== + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.44 + 3.1.43 ====== From 636b3e00d5f3383ecf7aff1388796e290cd5a126 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 2 Jan 2025 03:40:48 -0500 Subject: [PATCH 1093/1392] Test Python 3.7 on Ubuntu 22.04, and add Ubuntu 3.13 This is analogous to the gitdb test workflow and `setup.py` updates in https://github.com/gitpython-developers/gitdb/pull/114. 1. Testing 3.7 on 22.04 rather than 24.04 should fix the problem where it fails because Python 3.7 is not available. 2. Adding Ubuntu 3.13 to CI may help reveal if there are 3.13-specific problems with smmap. 3. smmap seems to be working on Python 3.13 (which is a stable Python release) and there are no specific expected problems with it, so this adds it to the list of supported releases. In particular, this change, due to (1), fixes the current CI failure for smmap observed in f31bfa3. --- .github/workflows/pythonpackage.yml | 10 +++++++--- setup.py | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index fd73d6dd6..ab0a8ef19 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -7,13 +7,17 @@ on: [push, pull_request, workflow_dispatch] jobs: build: - - runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] include: - experimental: false + - os: ubuntu-latest + - python-version: "3.7" + os: ubuntu-22.04 + + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.experimental }} steps: diff --git a/setup.py b/setup.py index d7e18420b..7deb1788e 100755 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3 :: Only", ], long_description=long_description, From 9429be69c85442e744ef697bd79bad3fb4236e0a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 2 Jan 2025 04:21:49 -0500 Subject: [PATCH 1094/1392] Special-case Python 3.7 for OSes we can install it on This is analogous to the 3.7-related CI change in gitdb that was part of https://github.com/gitpython-developers/gitdb/pull/114, as to part of https://github.com/gitpython-developers/smmap/pull/58. Since some tests are not yet passing on 3.13, this does not add 3.13 to CI, nor to the documentation of supported versions in `setup.py`. Note that the list there is not enforced; GitPython can already be installed on Python 3.13 and probably *mostly* works. (See https://github.com/gitpython-developers/GitPython/pull/1955 for details on other changes that should be made to fully support running GitPython on Python 3.13.) --- .github/workflows/pythonpackage.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 747db62f0..deebe9e11 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -11,15 +11,19 @@ permissions: jobs: build: strategy: - fail-fast: false matrix: - os: ["ubuntu-22.04", "macos-latest", "windows-latest"] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] - exclude: - - os: "macos-latest" - python-version: "3.7" + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] include: - experimental: false + - os: ubuntu-22.04 + python-version: "3.7" + experimental: false + - os: windows-latest + python-version: "3.7" + experimental: false + + fail-fast: false runs-on: ${{ matrix.os }} From affab8eda6a716363bc36c703de305676afc4ae3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:14:20 +0000 Subject: [PATCH 1095/1392] Bump Vampire/setup-wsl from 3.1.1 to 4.0.0 Bumps [Vampire/setup-wsl](https://github.com/vampire/setup-wsl) from 3.1.1 to 4.0.0. - [Release notes](https://github.com/vampire/setup-wsl/releases) - [Commits](https://github.com/vampire/setup-wsl/compare/v3.1.1...v4.0.0) --- updated-dependencies: - dependency-name: Vampire/setup-wsl dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index deebe9e11..b8e6391a1 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -44,7 +44,7 @@ jobs: - name: Set up WSL (Windows) if: startsWith(matrix.os, 'windows') - uses: Vampire/setup-wsl@v3.1.1 + uses: Vampire/setup-wsl@v4.0.0 with: distribution: Alpine additional-packages: bash From 01e40a7b06c4ba3d0cf937ba0974949392446b51 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 2 Jan 2025 04:38:44 -0500 Subject: [PATCH 1096/1392] Do everything in the venv in the Alpine test --- .github/workflows/alpine-test.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 2c1eed391..8c00b1086 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -16,10 +16,10 @@ jobs: steps: - name: Prepare Alpine Linux run: | - apk add sudo git git-daemon python3 py3-pip + apk add sudo git git-daemon python3 py3-pip py3-setuptools py3-virtualenv py3-wheel echo 'Defaults env_keep += "CI GITHUB_* RUNNER_*"' >/etc/sudoers.d/ci_env addgroup -g 127 docker - adduser -D -u 1001 runner + adduser -D -u 1001 runner # TODO: Check if this still works on GHA as intended. adduser runner docker shell: sh -exo pipefail {0} # Run this as root, not the "runner" user. @@ -50,17 +50,14 @@ jobs: . .venv/bin/activate printf '%s=%s\n' 'PATH' "$PATH" 'VIRTUAL_ENV' "$VIRTUAL_ENV" >>"$GITHUB_ENV" - - name: Update PyPA packages - run: | - # Get the latest pip, wheel, and prior to Python 3.12, setuptools. - python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel - - name: Install project and test dependencies run: | + . .venv/bin/activate pip install ".[test]" - name: Show version and platform information run: | + . .venv/bin/activate uname -a command -v git python git version @@ -69,4 +66,5 @@ jobs: - name: Test with pytest run: | + . .venv/bin/activate pytest --color=yes -p no:sugar --instafail -vv From 0300de986ef78a4e7a5562638592f5e91bfd8fa7 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 2 Jan 2025 05:24:47 -0500 Subject: [PATCH 1097/1392] Instrument `handle_process_output` test To try to find the bug that causes it to fail on Cygwin on CI, but not on other systems on CI, and not locally on Cygwin. It looks like there's an extra line being read from stderr when the test fails, so let's examine the lines themselves. --- test/test_git.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/test/test_git.py b/test/test_git.py index 94e68ecf0..3f9687dfe 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -762,14 +762,17 @@ def test_environment(self, rw_dir): def test_handle_process_output(self): from git.cmd import handle_process_output, safer_popen - line_count = 5002 - count = [None, 0, 0] + expected_line_count = 5002 + line_counts = [None, 0, 0] + lines = [None, [], []] - def counter_stdout(line): - count[1] += 1 + def stdout_handler(line): + line_counts[1] += 1 + lines[1].append(line) - def counter_stderr(line): - count[2] += 1 + def stderr_handler(line): + line_counts[2] += 1 + lines[2].append(line) cmdline = [ sys.executable, @@ -784,10 +787,10 @@ def counter_stderr(line): shell=False, ) - handle_process_output(proc, counter_stdout, counter_stderr, finalize_process) + handle_process_output(proc, stdout_handler, stderr_handler, finalize_process) - self.assertEqual(count[1], line_count) - self.assertEqual(count[2], line_count) + self.assertEqual(line_counts[1], expected_line_count, repr(lines[1])) + self.assertEqual(line_counts[2], expected_line_count, repr(lines[2])) def test_execute_kwargs_set_agrees_with_method(self): parameter_names = inspect.signature(cmd.Git.execute).parameters.keys() From a0f8425c94992bdf3fdde9cbf7c3c7f9dc12e52c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 2 Jan 2025 05:28:31 -0500 Subject: [PATCH 1098/1392] Slightly simplify and clarify instrumented code --- test/test_git.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/test/test_git.py b/test/test_git.py index 3f9687dfe..274511f8d 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -763,16 +763,13 @@ def test_handle_process_output(self): from git.cmd import handle_process_output, safer_popen expected_line_count = 5002 - line_counts = [None, 0, 0] - lines = [None, [], []] + actual_lines = [None, [], []] def stdout_handler(line): - line_counts[1] += 1 - lines[1].append(line) + actual_lines[1].append(line) def stderr_handler(line): - line_counts[2] += 1 - lines[2].append(line) + actual_lines[2].append(line) cmdline = [ sys.executable, @@ -789,8 +786,8 @@ def stderr_handler(line): handle_process_output(proc, stdout_handler, stderr_handler, finalize_process) - self.assertEqual(line_counts[1], expected_line_count, repr(lines[1])) - self.assertEqual(line_counts[2], expected_line_count, repr(lines[2])) + self.assertEqual(len(actual_lines[1]), expected_line_count, repr(actual_lines[1])) + self.assertEqual(len(actual_lines[2]), expected_line_count, repr(actual_lines[2])) def test_execute_kwargs_set_agrees_with_method(self): parameter_names = inspect.signature(cmd.Git.execute).parameters.keys() From 4aad37a303ca51a594700976b04e17f0f835d61d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 2 Jan 2025 06:00:23 -0500 Subject: [PATCH 1099/1392] Improve environment isolation in Cygwin and Alpine Linux on CI The main change here is to the Cygwin test workflow, which now runs the tests in a venv (a Python virtual environment), to avoid mixing up our intended dependencies with other files installed by Python related packages on the system. This should either fix the problem where `test_handle_process_output` reports an extra line in stderr for the `cat_file.py` subprocess on CI or Cygwin, or at least make it somewhat easier to continue investigating the problem. I think this change is also valuable for its own sake. The connection to the `test_handle_process_output` failure is that the extra line in stderr appears at the beginning and is an error message about a missing import needed for subprocess code coverage. With the recently added more detailed error reporting, it shows: self.assertEqual(line_counts[1], expected_line_count, repr(lines[1])) > self.assertEqual(line_counts[2], expected_line_count, repr(lines[2])) E AssertionError: 5003 != 5002 : ['pytest-cov: Failed to setup subprocess coverage. Environ: {\'COV_CORE_SOURCE\': \'git\', \'COV_CORE_CONFIG\': \':\', \'COV_CORE_DATAFILE\': \'/cygdrive/d/a/GitPython/GitPython/.coverage\'} Exception: ModuleNotFoundError("No module named \'iniconfig\'")\n', 'From github.com:jantman/gitpython_issue_301\n', ' = [up to date] master -> origin/master\n', ' = [up to date] testcommit1 -> origin/testcommit1\n', ' = [up to date] testcommit10 -> origin/testcommit10\n', ' = [up to date] testcommit100 -> origin/testcommit100\n', ' = [up to date] testcommit1000 -> origin/testcommit1000\n', ' = [up to date] testcommit1001 -> origin/testcommit1001\n', ' = [up to date] testcommit1002 -> origin/testcommit1002\n', ' = [up to date] testcommit1003 -> origin/testcommit1003\n', ' = [up to date] testcommit1004 -> origin/testcommit1004\n', ' = [up to date] testcommit1005 -> origin/testcommit1005\n', ' = [up to date] testcommit test/test_git.py:793: AssertionError This could possibly be worked around by attempting to install a package to provide `iniconfig`, by configuring `pytest-cov` in a way that does not require it, or by temporarily disabling code coverage reports on Cygwin. Before exploring those or other options, this change seeks to prepare a more isolated environment in which different package versions are more likely to work properly together. In addition to that change to the Cygwin workflow, this also changes the way the Alpine Linux test workflow is made to use a venv, using the technique that is used here, and undoing some changes in the Alpine Linux workflow that should not be necessary. The reason for making that change together with this Cygwin change is that if either does not work as expected, it might shed light on what is going wrong with the other. Although the same technique is used on Cygwin and in Alpine Linux, it looks a little different, because the environment variable on Cygwin is `BASH_ENV` (since script steps are run in `bash`), while the environment variable is `ENV` (since script steps are run in `busybox sh`) and this must also be allowed to pass through `sudo` (since `sh`, which is `busybox sh` on this system, is being invoked through `sudo`). --- .github/workflows/alpine-test.yml | 15 ++++++++------- .github/workflows/cygwin-test.yml | 6 +++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 8c00b1086..ca141cf03 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -16,8 +16,8 @@ jobs: steps: - name: Prepare Alpine Linux run: | - apk add sudo git git-daemon python3 py3-pip py3-setuptools py3-virtualenv py3-wheel - echo 'Defaults env_keep += "CI GITHUB_* RUNNER_*"' >/etc/sudoers.d/ci_env + apk add sudo git git-daemon python3 py3-pip py3-virtualenv + echo 'Defaults env_keep += "CI ENV GITHUB_* RUNNER_*"' >/etc/sudoers.d/ci_env addgroup -g 127 docker adduser -D -u 1001 runner # TODO: Check if this still works on GHA as intended. adduser runner docker @@ -47,17 +47,19 @@ jobs: - name: Set up virtualenv run: | python -m venv .venv - . .venv/bin/activate - printf '%s=%s\n' 'PATH' "$PATH" 'VIRTUAL_ENV' "$VIRTUAL_ENV" >>"$GITHUB_ENV" + echo 'ENV=.venv/bin/activate' >> "$GITHUB_ENV" # ENV (not BASH_ENV) for BusyBox sh. + + - name: Update PyPA packages + run: | + # Get the latest pip, wheel, and prior to Python 3.12, setuptools. + python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel - name: Install project and test dependencies run: | - . .venv/bin/activate pip install ".[test]" - name: Show version and platform information run: | - . .venv/bin/activate uname -a command -v git python git version @@ -66,5 +68,4 @@ jobs: - name: Test with pytest run: | - . .venv/bin/activate pytest --color=yes -p no:sugar --instafail -vv diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index bde4ea659..ebe50240d 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -55,10 +55,10 @@ jobs: # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig - - name: Ensure the "pip" command is available + - name: Set up virtualenv run: | - # This is used unless, and before, an updated pip is installed. - ln -s pip3 /usr/bin/pip + python -m venv .venv + echo 'BASH_ENV=.venv/bin/activate' >>"$GITHUB_ENV" - name: Update PyPA packages run: | From 39cd608b762256663b862224bcb46bdb2fc18817 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 2 Jan 2025 06:28:48 -0500 Subject: [PATCH 1100/1392] Put back explicit venv activation in Alpine Linux `busybox sh` does not appear to read commands from a file whose path is given as the value of `$ENV`, in this situation. I think I may have misunderstood that; the documentation does not say much about it and maybe, in Almquist-style shells, it is only read by interactive shells? I am not sure. This removes everything about `ENV` and activates the venv in each step where the venv should be used. The good news is that the technique did work fully in Cygwin: not only did `BASH_ENV` work (which was not much in doubt), but using a virtual environment for all steps that run test code on Cygwin fixed the problem and allowed all tests to pass. That seems to have been the reason I didn't reproduce the problem locally: on my local system I always use a venv in Cygwin since I would otherwise not have adequate isolation. Thus, this commit changes only the Alpine workflow and not the Cygwin workflow. --- .github/workflows/alpine-test.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index ca141cf03..6dc62f596 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -17,7 +17,7 @@ jobs: - name: Prepare Alpine Linux run: | apk add sudo git git-daemon python3 py3-pip py3-virtualenv - echo 'Defaults env_keep += "CI ENV GITHUB_* RUNNER_*"' >/etc/sudoers.d/ci_env + echo 'Defaults env_keep += "CI GITHUB_* RUNNER_*"' >/etc/sudoers.d/ci_env addgroup -g 127 docker adduser -D -u 1001 runner # TODO: Check if this still works on GHA as intended. adduser runner docker @@ -47,19 +47,21 @@ jobs: - name: Set up virtualenv run: | python -m venv .venv - echo 'ENV=.venv/bin/activate' >> "$GITHUB_ENV" # ENV (not BASH_ENV) for BusyBox sh. - name: Update PyPA packages run: | # Get the latest pip, wheel, and prior to Python 3.12, setuptools. + . .venv/bin/activate python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel - name: Install project and test dependencies run: | + . .venv/bin/activate pip install ".[test]" - name: Show version and platform information run: | + . .venv/bin/activate uname -a command -v git python git version @@ -68,4 +70,5 @@ jobs: - name: Test with pytest run: | + . .venv/bin/activate pytest --color=yes -p no:sugar --instafail -vv From 8a05390925ef416736ce9c0be8569977ca48fa07 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 2 Jan 2025 07:22:11 -0500 Subject: [PATCH 1101/1392] Tune CI matrix adjustments so reports are clearer Since #1987, test jobs from `pythonpackage.yml` appear in an unintuitive order, and some show an extra bool matrix variable in their names while others don't (this corresponds to `experimental`, which was always set to some value, but was set in different ways). This fixes that by: - Listing all tested versions, rather than introducing some in an `include` key. (The `include:`-introduced jobs didn't distinguish between originally-present matrix variables and those that are introduced based on the values of the original ones.) - Replacing `os` with `os-type`, which has only the first part of the value for `runs-on:` (e.g., `ubuntu`), and adding `os-ver` to each matrix job, defaulting it to `latest`, but using `22.04` for Python 3.7 on Ubuntu. This should also naturally extend to adding 3.13, with or without setting `continue-on-error` to temporarily work around the problems obseved in #1955, but nothing 3.13-related is done in this commit. --- .github/workflows/pythonpackage.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index b8e6391a1..aff6354f4 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -12,20 +12,21 @@ jobs: build: strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - include: - - experimental: false - - os: ubuntu-22.04 + os-type: [ubuntu, macos, windows] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + exclude: + - os-type: macos python-version: "3.7" - experimental: false - - os: windows-latest + include: + - os-ver: latest + - os-type: ubuntu python-version: "3.7" - experimental: false + os-ver: "22.04" + - experimental: false fail-fast: false - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os-type }}-${{ matrix.os-ver }} defaults: run: @@ -43,7 +44,7 @@ jobs: allow-prereleases: ${{ matrix.experimental }} - name: Set up WSL (Windows) - if: startsWith(matrix.os, 'windows') + if: matrix.os-type == 'windows' uses: Vampire/setup-wsl@v4.0.0 with: distribution: Alpine @@ -80,7 +81,7 @@ jobs: # For debugging hook tests on native Windows systems that may have WSL. - name: Show bash.exe candidates (Windows) - if: startsWith(matrix.os, 'windows') + if: matrix.os-type == 'windows' run: | set +e bash.exe -c 'printenv WSL_DISTRO_NAME; uname -a' From 41377d5254d3e4a03a7edd5adc8fd4b5a0767210 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 2 Jan 2025 07:39:19 -0500 Subject: [PATCH 1102/1392] Change test job keys from `build` to `test` This goes a bit further in the direction of the preceding commit, making CI reports/logs a bit more intuitive. --- .github/workflows/alpine-test.yml | 2 +- .github/workflows/cygwin-test.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 6dc62f596..bd09a939b 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -3,7 +3,7 @@ name: test-alpine on: [push, pull_request, workflow_dispatch] jobs: - build: + test: runs-on: ubuntu-latest container: diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index ebe50240d..575fb26ef 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -3,7 +3,7 @@ name: test-cygwin on: [push, pull_request, workflow_dispatch] jobs: - build: + test: runs-on: windows-latest strategy: diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index aff6354f4..9225624f0 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -9,7 +9,7 @@ permissions: contents: read jobs: - build: + test: strategy: matrix: os-type: [ubuntu, macos, windows] From 73ddb22f5c06fc3f09addf9be176d569d770b469 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 4 Jan 2025 09:55:50 -0500 Subject: [PATCH 1103/1392] Continue testing Python 3.9 on CI but unpin 3.9.16-1 We pinned Python 3.9.16 on Cygwin CI in #1814 (by requiring 3.9.16-1 as the exact version of the `python39` Cygwin package, along with other supporting changes). We did this to solve a problem where Python 3.9.18-1, which contained a bug that broke GitPython CI (and various other software), would be selected. Version 3.9.18-1 was marked back to being a "test" package shortly after the bug was reported, and was subsequently removed altogether from the Cygwin repositories. Because the affected package version effectively no longer exists, and because this issue is known and a non-"test" version still affected by it is very unlikely to be released in the future, this pinning has been decisively unnecessary for some time, though still not harmful. This commit undoes the pinning, so that the `python39` package can be installed at a higher version if one becomes available. This serves two purposes. - There is work under way in porting Python 3.12 to Cygwin. To test this with GitPython (either while it is in development or later), it will be useful to turn the Cygwin test job into a matrix job definition, generating two jobs, one for Python 3.9 and one for Python 3.12. Since 3.12 will probably not benefit from pinning, dropping pinning simplifies this. - If the port of Python 3.12 to Cygwin is successful, it might lead to a solution to the but that currently keeps 3.9.18 from being made available for Cygwin. In that case, another 3.9.18-* Cygwin package would be released, which we would want to use. Although this is uncertain, the change is a simplification, so I think it is reasonable to do now. Note that the pinning being undone here only affects the distinction between different 3.9.* versions. `python39` and `python312` are different Cygwin packages altogether, with correspondingly different `python39-*` and `python312-*` associated packages; this is not unpinning Python 3.9 in a way that would cause Python 3.12 to be selected instead of it. --- .github/workflows/cygwin-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 575fb26ef..d42eb0587 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -30,7 +30,7 @@ jobs: - name: Set up Cygwin uses: egor-tensin/setup-cygwin@v4 with: - packages: python39=3.9.16-1 python39-pip python39-virtualenv git + packages: python39 python39-pip python39-virtualenv git - name: Arrange for verbose output run: | From 85d72ef55eb3ba64f5db6016198724ec45769961 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 5 Jan 2025 02:38:34 -0500 Subject: [PATCH 1104/1392] Test Python 3.13 on Ubuntu and macOS on CI But not Windows yet (#1955). --- .github/workflows/pythonpackage.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 9225624f0..245844972 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -13,10 +13,12 @@ jobs: strategy: matrix: os-type: [ubuntu, macos, windows] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] exclude: - os-type: macos - python-version: "3.7" + python-version: "3.7" # Not available for the ARM-based macOS runners. + - os-type: windows + python-version: "3.13" # FIXME: Fix and enable Python 3.13 on Windows (#1955). include: - os-ver: latest - os-type: ubuntu From 13b8c9b8d97ed499076fdf9e75fe51a2cd92562a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 5 Jan 2025 03:17:00 -0500 Subject: [PATCH 1105/1392] Add SECURITY.md, referencing the GitPython one See https://github.com/gitpython-developers/gitdb/issues/116. --- SECURITY.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..9e0c0d16a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +See [GitPython](https://github.com/gitpython-developers/GitPython/blob/main/SECURITY.md). Vulnerabilities found in `smmap` can be reported there. From 26209528a0303e47c88c174184adbf25d206a824 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 5 Jan 2025 03:21:33 -0500 Subject: [PATCH 1106/1392] Add SECURITY.md, referencing GitPython's Along with https://github.com/gitpython-developers/smmap/pull/59 and a forthcoming related PR in GitPython, this will fix #116. --- SECURITY.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..95389ff6e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +See [GitPython](https://github.com/gitpython-developers/GitPython/blob/main/SECURITY.md). Vulnerabilities found in `gitdb` can be reported there. From b20de09016ce221943a7bc4c7b67be5bacad9a15 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 5 Jan 2025 03:24:28 -0500 Subject: [PATCH 1107/1392] Affirm that gitdb and smmap advisories can also be created This expands `SECURITY.md` to affirm the claims in the new `SECURITY.md` files in gitdb and smmap that vulnerabilities found in them can be reported in the GitPython repository with the same link as one would use to report a GitPython vulnerability, as well as to note how the distinction between affected package can be specified when it is known at the time a vulnerability is reported. Along with https://github.com/gitpython-developers/smmap/pull/59 and https://github.com/gitpython-developers/gitdb/pull/117, this fixes https://github.com/gitpython-developers/gitdb/issues/116. --- SECURITY.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index d39425b70..3f7d9f27e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,4 +11,6 @@ Only the latest version of GitPython can receive security updates. If a vulnerab ## Reporting a Vulnerability -Please report private portions of a vulnerability to . Doing so helps to receive updates and collaborate on the matter, without disclosing it publicliy right away. +Please report private portions of a vulnerability to . Doing so helps to receive updates and collaborate on the matter, without disclosing it publicly right away. + +Vulnerabilities in GitPython's dependencies [gitdb](https://github.com/gitpython-developers/gitdb/blob/main/SECURITY.md) or [smmap](https://github.com/gitpython-developers/smmap/blob/main/SECURITY.md), which primarily exist to support GitPython, can be reported here as well, at that same link. The affected package (`GitPython`, `gitdb`, or `smmap`) can be included in the report, if known. From 55f36e64d79e0e8acc8f8763af5e7b18a3b214f6 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 5 Jan 2025 11:51:31 -0500 Subject: [PATCH 1108/1392] Fix links to gitdb and smmap `SECURITY.md` files The links in #1991 did not work, as I got the branch names wrong. --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 3f7d9f27e..0aea34845 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -13,4 +13,4 @@ Only the latest version of GitPython can receive security updates. If a vulnerab Please report private portions of a vulnerability to . Doing so helps to receive updates and collaborate on the matter, without disclosing it publicly right away. -Vulnerabilities in GitPython's dependencies [gitdb](https://github.com/gitpython-developers/gitdb/blob/main/SECURITY.md) or [smmap](https://github.com/gitpython-developers/smmap/blob/main/SECURITY.md), which primarily exist to support GitPython, can be reported here as well, at that same link. The affected package (`GitPython`, `gitdb`, or `smmap`) can be included in the report, if known. +Vulnerabilities in GitPython's dependencies [gitdb](https://github.com/gitpython-developers/gitdb/blob/master/SECURITY.md) or [smmap](https://github.com/gitpython-developers/smmap/blob/master/SECURITY.md), which primarily exist to support GitPython, can be reported here as well, at that same link. The affected package (`GitPython`, `gitdb`, or `smmap`) can be included in the report, if known. From 4fe56572894f9668c1ffd0808c96aed27c65e584 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 10:38:13 +0000 Subject: [PATCH 1109/1392] Bump gitdb/ext/smmap from `f31bfa3` to `8f82e6c` Bumps [gitdb/ext/smmap](https://github.com/gitpython-developers/smmap) from `f31bfa3` to `8f82e6c`. - [Release notes](https://github.com/gitpython-developers/smmap/releases) - [Commits](https://github.com/gitpython-developers/smmap/compare/f31bfa378c8840d38d31e7e11ef2b84f191a491e...8f82e6c19661f9b735cc55cc89031a189e408894) --- updated-dependencies: - dependency-name: gitdb/ext/smmap dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- gitdb/ext/smmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index f31bfa378..8f82e6c19 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit f31bfa378c8840d38d31e7e11ef2b84f191a491e +Subproject commit 8f82e6c19661f9b735cc55cc89031a189e408894 From 80096b95e119e90c9324f5ba898705fda4581c84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 13:38:24 +0000 Subject: [PATCH 1110/1392] Bump Vampire/setup-wsl from 4.0.0 to 4.1.0 Bumps [Vampire/setup-wsl](https://github.com/vampire/setup-wsl) from 4.0.0 to 4.1.0. - [Release notes](https://github.com/vampire/setup-wsl/releases) - [Commits](https://github.com/vampire/setup-wsl/compare/v4.0.0...v4.1.0) --- updated-dependencies: - dependency-name: Vampire/setup-wsl dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 245844972..68b4fd8f9 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -47,7 +47,7 @@ jobs: - name: Set up WSL (Windows) if: matrix.os-type == 'windows' - uses: Vampire/setup-wsl@v4.0.0 + uses: Vampire/setup-wsl@v4.1.0 with: distribution: Alpine additional-packages: bash From fb102cfc5a86155d2c8b8f10cf5957f3f5d9e435 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 13:45:04 +0000 Subject: [PATCH 1111/1392] Bump git/ext/gitdb from `775cfe8` to `9e68ea1` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `775cfe8` to `9e68ea1`. - [Release notes](https://github.com/gitpython-developers/gitdb/releases) - [Commits](https://github.com/gitpython-developers/gitdb/compare/775cfe8299ea5474f605935469359a9d1cdb49dc...9e68ea1687bbda84776c3605b96bb0b43e846bea) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 775cfe829..9e68ea168 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 775cfe8299ea5474f605935469359a9d1cdb49dc +Subproject commit 9e68ea1687bbda84776c3605b96bb0b43e846bea From e4f1aa71dd255583ff19c1bd40410e94da8e15af Mon Sep 17 00:00:00 2001 From: Frank Lichtenheld Date: Thu, 9 Jan 2025 17:57:51 +0100 Subject: [PATCH 1112/1392] Repo.rev_parse: Handle ^{commit} correctly This should resolve to commit object. Fixes: #1995 Signed-off-by: Frank Lichtenheld --- git/repo/fun.py | 8 +++++++- test/test_repo.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/git/repo/fun.py b/git/repo/fun.py index 182cf82ed..125ba5936 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -301,7 +301,13 @@ def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: # Handle type. if output_type == "commit": - pass # Default. + obj = cast("TagObject", obj) + if obj and obj.type == "tag": + obj = deref_tag(obj) + else: + # Cannot do anything for non-tags. + pass + # END handle tag elif output_type == "tree": try: obj = cast(AnyGitObject, obj) diff --git a/test/test_repo.py b/test/test_repo.py index e38da5bb6..bfa1bbb78 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -1064,9 +1064,9 @@ def test_rev_parse(self): # TODO: Dereference tag into a blob 0.1.7^{blob} - quite a special one. # Needs a tag which points to a blob. - # ref^0 returns commit being pointed to, same with ref~0, and ^{} + # ref^0 returns commit being pointed to, same with ref~0, ^{}, and ^{commit} tag = rev_parse("0.1.4") - for token in ("~0", "^0", "^{}"): + for token in ("~0", "^0", "^{}", "^{commit}"): self.assertEqual(tag.object, rev_parse("0.1.4%s" % token)) # END handle multiple tokens From 7751d0b98920f9fc74b2f109e92cb0abf3a98b15 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Fri, 10 Jan 2025 13:52:56 -0500 Subject: [PATCH 1113/1392] Fuzzing: Fix broken test for Git submodule handling Ensured submodule names, paths, and commit messages are sanitized to avoid invalid states that are expected to cause exceptions and should not halt the fuzzer. In particular, the changes here: - Sanitized inputs for submodule names, paths, and commit messages. - Added validation for submodule SHA and path integrity. --- fuzzing/fuzz-targets/fuzz_submodule.py | 59 ++++++++++++++++++-------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py index d22b0aa5b..afa653d0d 100644 --- a/fuzzing/fuzz-targets/fuzz_submodule.py +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -9,11 +9,17 @@ get_max_filename_length, ) -# Setup the git environment +# Setup the Git environment setup_git_environment() from git import Repo, GitCommandError, InvalidGitRepositoryError +def sanitize_input(input_str, max_length=255): + """Sanitize and truncate inputs to avoid invalid Git operations.""" + sanitized = "".join(ch for ch in input_str if ch.isalnum() or ch in ("-", "_", ".")) + return sanitized[:max_length] + + def TestOneInput(data): fdp = atheris.FuzzedDataProvider(data) @@ -24,12 +30,23 @@ def TestOneInput(data): try: with tempfile.TemporaryDirectory() as submodule_temp_dir: sub_repo = Repo.init(submodule_temp_dir, bare=fdp.ConsumeBool()) - sub_repo.index.commit(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))) + commit_message = sanitize_input(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))) + sub_repo.index.commit(commit_message) - submodule_name = fdp.ConsumeUnicodeNoSurrogates( - fdp.ConsumeIntInRange(1, max(1, get_max_filename_length(repo.working_tree_dir))) + submodule_name = sanitize_input( + fdp.ConsumeUnicodeNoSurrogates( + fdp.ConsumeIntInRange(1, get_max_filename_length(repo.working_tree_dir)) + ) ) - submodule_path = os.path.join(repo.working_tree_dir, submodule_name) + + submodule_path = os.path.relpath( + os.path.join(repo.working_tree_dir, submodule_name), + start=repo.working_tree_dir, + ) + + # Ensure submodule_path is valid + if not submodule_name or submodule_name.startswith("/") or ".." in submodule_name: + return -1 # Reject invalid input so they are not added to the corpus submodule = repo.create_submodule(submodule_name, submodule_path, url=sub_repo.git_dir) repo.index.commit("Added submodule") @@ -39,25 +56,38 @@ def TestOneInput(data): value_length = fdp.ConsumeIntInRange(1, max(1, fdp.remaining_bytes())) writer.set_value( - fdp.ConsumeUnicodeNoSurrogates(key_length), fdp.ConsumeUnicodeNoSurrogates(value_length) + sanitize_input(fdp.ConsumeUnicodeNoSurrogates(key_length)), + sanitize_input(fdp.ConsumeUnicodeNoSurrogates(value_length)), ) writer.release() - submodule.update(init=fdp.ConsumeBool(), dry_run=fdp.ConsumeBool(), force=fdp.ConsumeBool()) + submodule.update( + init=fdp.ConsumeBool(), + dry_run=fdp.ConsumeBool(), + force=fdp.ConsumeBool(), + ) + submodule_repo = submodule.module() - new_file_name = fdp.ConsumeUnicodeNoSurrogates( - fdp.ConsumeIntInRange(1, max(1, get_max_filename_length(submodule_repo.working_tree_dir))) + new_file_name = sanitize_input( + fdp.ConsumeUnicodeNoSurrogates( + fdp.ConsumeIntInRange(1, get_max_filename_length(submodule_repo.working_tree_dir)) + ) ) new_file_path = os.path.join(submodule_repo.working_tree_dir, new_file_name) with open(new_file_path, "wb") as new_file: new_file.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512))) + submodule_repo.index.add([new_file_path]) submodule_repo.index.commit("Added new file to submodule") repo.submodule_update(recursive=fdp.ConsumeBool()) - submodule_repo.head.reset(commit="HEAD~1", working_tree=fdp.ConsumeBool(), head=fdp.ConsumeBool()) - # Use fdp.PickValueInList to ensure at least one of 'module' or 'configuration' is True + submodule_repo.head.reset( + commit="HEAD~1", + working_tree=fdp.ConsumeBool(), + head=fdp.ConsumeBool(), + ) + module_option_value, configuration_option_value = fdp.PickValueInList( [(True, False), (False, True), (True, True)] ) @@ -82,12 +112,7 @@ def TestOneInput(data): ): return -1 except Exception as e: - if isinstance(e, ValueError) and "embedded null byte" in str(e): - return -1 - elif isinstance(e, OSError) and "File name too long" in str(e): - return -1 - else: - return handle_exception(e) + return handle_exception(e) def main(): From b4fd74ce8e28c372c511db2e0a491fa8b67c93f4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 26 Jan 2025 11:51:11 -0500 Subject: [PATCH 1114/1392] Improve description of backoff sequence in db.loose The sequence of backoff wait times used in `gitdb.db.loose` is quadratic rather than exponential, as discussed in: https://github.com/gitpython-developers/gitdb/pull/115#discussion_r1903215598 This corrects the variable name by making it more general, and the comment by having it explicitly describe the backoff as quadratic. This is conceptually related to GitoxideLabs/gitoxide#1815, but this is a non-breaking change, as no interfaces are affected: only a local variable and comment. --- gitdb/db/loose.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gitdb/db/loose.py b/gitdb/db/loose.py index 03d387e86..e6765cdeb 100644 --- a/gitdb/db/loose.py +++ b/gitdb/db/loose.py @@ -230,16 +230,16 @@ def store(self, istream): # end rename only if needed # Ensure rename is actually done and file is stable - # Retry up to 14 times - exponential wait & retry in ms. + # Retry up to 14 times - quadratic wait & retry in ms. # The total maximum wait time is 1000ms, which should be vastly enough for the # OS to return and commit the file to disk. - for exp_backoff_ms in [1, 4, 9, 16, 25, 36, 49, 64, 81, 100, 121, 144, 169, 181]: + for backoff_ms in [1, 4, 9, 16, 25, 36, 49, 64, 81, 100, 121, 144, 169, 181]: with suppress(PermissionError): # make sure its readable for all ! It started out as rw-- tmp file # but needs to be rwrr chmod(obj_path, self.new_objects_mode) break - time.sleep(exp_backoff_ms / 1000.0) + time.sleep(backoff_ms / 1000.0) else: raise PermissionError( "Impossible to apply `chmod` to file {}".format(obj_path) From 5c547f33063d811f445c82de19b0d2f6aad0e995 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 13:17:13 +0000 Subject: [PATCH 1115/1392] Bump git/ext/gitdb from `9e68ea1` to `f36c0cc` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `9e68ea1` to `f36c0cc`. - [Release notes](https://github.com/gitpython-developers/gitdb/releases) - [Commits](https://github.com/gitpython-developers/gitdb/compare/9e68ea1687bbda84776c3605b96bb0b43e846bea...f36c0cc42ea2f529291e441073f74e920988d4d2) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 9e68ea168..f36c0cc42 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 9e68ea1687bbda84776c3605b96bb0b43e846bea +Subproject commit f36c0cc42ea2f529291e441073f74e920988d4d2 From 9e22a4a5811157e59a61778285d79b686bbec9ad Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 21 Feb 2025 05:17:30 -0500 Subject: [PATCH 1116/1392] Run `python3.9` explicitly on Cygwin CI In case somehow another one leaked in. --- .github/workflows/cygwin-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index d42eb0587..0a66a9b46 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -57,7 +57,7 @@ jobs: - name: Set up virtualenv run: | - python -m venv .venv + python3.9 -m venv .venv echo 'BASH_ENV=.venv/bin/activate' >>"$GITHUB_ENV" - name: Update PyPA packages From f401b1020f4177b217d679d71a41df28754bef6f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 21 Feb 2025 05:19:56 -0500 Subject: [PATCH 1117/1392] Switch back to cygwin/cygwin-install-action Since we don't need pinning, and this avoids installing and using the chocolatey package manager, which we're not using for anything else. --- .github/workflows/cygwin-test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 0a66a9b46..fbf9b8307 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -15,7 +15,7 @@ jobs: defaults: run: - shell: C:\tools\cygwin\bin\bash.exe --login --norc -eo pipefail -o igncr "{0}" + shell: C:\cygwin\bin\bash.exe --login --norc -eo pipefail -o igncr "{0}" steps: - name: Force LF line endings @@ -27,10 +27,11 @@ jobs: with: fetch-depth: 0 - - name: Set up Cygwin - uses: egor-tensin/setup-cygwin@v4 + - name: Install Cygwin + uses: cygwin/cygwin-install-action@v5 with: packages: python39 python39-pip python39-virtualenv git + add-to-path: false # No need to change $PATH outside the Cygwin environment. - name: Arrange for verbose output run: | From 4605dd690b6ebe4c0c07f3910607aae407d6070e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 21 Feb 2025 05:30:50 -0500 Subject: [PATCH 1118/1392] Install pip in venv in separate step --- .github/workflows/cygwin-test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index fbf9b8307..a2e7f6b2d 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -58,9 +58,13 @@ jobs: - name: Set up virtualenv run: | - python3.9 -m venv .venv + python3.9 -m venv --without-pip .venv echo 'BASH_ENV=.venv/bin/activate' >>"$GITHUB_ENV" + - name: Install pip in virtualenv + run: | + python -m ensurepip + - name: Update PyPA packages run: | # Get the latest pip, wheel, and prior to Python 3.12, setuptools. From a5fee410ec8d40264d7a53cc5c4fd9deb0c97dd5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 21 Feb 2025 05:40:26 -0500 Subject: [PATCH 1119/1392] Install Cygwin package for wheel --- .github/workflows/cygwin-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index a2e7f6b2d..d4aee4544 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -30,7 +30,7 @@ jobs: - name: Install Cygwin uses: cygwin/cygwin-install-action@v5 with: - packages: python39 python39-pip python39-virtualenv git + packages: python39 python39-pip python39-virtualenv python39-wheel git add-to-path: false # No need to change $PATH outside the Cygwin environment. - name: Arrange for verbose output From a2b73cac7c252f3c35169dd8a752614614571d10 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 21 Feb 2025 05:45:30 -0500 Subject: [PATCH 1120/1392] Use the pip bootstrap script instead --- .github/workflows/cygwin-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index d4aee4544..585c419aa 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -30,7 +30,7 @@ jobs: - name: Install Cygwin uses: cygwin/cygwin-install-action@v5 with: - packages: python39 python39-pip python39-virtualenv python39-wheel git + packages: python39 python39-pip python39-virtualenv git add-to-path: false # No need to change $PATH outside the Cygwin environment. - name: Arrange for verbose output @@ -61,9 +61,9 @@ jobs: python3.9 -m venv --without-pip .venv echo 'BASH_ENV=.venv/bin/activate' >>"$GITHUB_ENV" - - name: Install pip in virtualenv + - name: Bootstrap pip in virtualenv run: | - python -m ensurepip + wget -qO- https://bootstrap.pypa.io/get-pip.py | python - name: Update PyPA packages run: | From d597fc949c48726b34c2130424a044020960e5f2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 21 Feb 2025 05:51:25 -0500 Subject: [PATCH 1121/1392] Install wget for Cygwin job to use to get pip bootstrap script --- .github/workflows/cygwin-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 585c419aa..278777907 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -30,7 +30,7 @@ jobs: - name: Install Cygwin uses: cygwin/cygwin-install-action@v5 with: - packages: python39 python39-pip python39-virtualenv git + packages: python39 python39-pip python39-virtualenv git wget add-to-path: false # No need to change $PATH outside the Cygwin environment. - name: Arrange for verbose output From 1277baa811e5500cf4aaae1569137fe1e6b4c3cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:53:19 +0000 Subject: [PATCH 1122/1392] Bump Vampire/setup-wsl from 4.1.0 to 4.1.1 Bumps [Vampire/setup-wsl](https://github.com/vampire/setup-wsl) from 4.1.0 to 4.1.1. - [Release notes](https://github.com/vampire/setup-wsl/releases) - [Commits](https://github.com/vampire/setup-wsl/compare/v4.1.0...v4.1.1) --- updated-dependencies: - dependency-name: Vampire/setup-wsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 68b4fd8f9..4850f252c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -47,7 +47,7 @@ jobs: - name: Set up WSL (Windows) if: matrix.os-type == 'windows' - uses: Vampire/setup-wsl@v4.1.0 + uses: Vampire/setup-wsl@v4.1.1 with: distribution: Alpine additional-packages: bash From 2ae697d02182d62ed1c17df370d7aa3a2b67cbce Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 5 Mar 2025 23:00:42 -0500 Subject: [PATCH 1123/1392] Mark `test_installation` xfail on Cygwin CI Together with #2007, this works around #2004, allowing all tests to pass on Cygwin CI. In #2007, installation of the environment in which tests run was fixed by downloading and running the `get-pip.py` bootstrap script. If we were to modify our helper that sets up the (separate) virtual environment in `test_installation` so that it does the same thing (or conditionally does so on CI, since the problem does not seem to happen in local installations), that would likely "fix" this more thoroughly, allowing the test to pass. But part of the goal of the installation test is to test that installation works in a typical environment on the platform it runs on. So it is not obivous that making it pass in that way would be an improvement compared to marking it `xfail` with the exception type that occurs due to #2004. So this just does that, for now. --- test/test_installation.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/test_installation.py b/test/test_installation.py index ae6472e98..a35826bd0 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -4,11 +4,19 @@ import ast import os import subprocess +import sys + +import pytest from test.lib import TestBase, VirtualEnvironment, with_rw_directory class TestInstallation(TestBase): + @pytest.mark.xfail( + sys.platform == "cygwin" and "CI" in os.environ, + reason="Trouble with pip on Cygwin CI, see issue #2004", + raises=subprocess.CalledProcessError, + ) @with_rw_directory def test_installation(self, rw_dir): venv = self._set_up_venv(rw_dir) From 54e1c1bfd14fc055fb8f7324154fd0658ac0d16f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Mar 2025 11:55:36 +0000 Subject: [PATCH 1124/1392] Bump Vampire/setup-wsl from 4.1.1 to 5.0.0 Bumps [Vampire/setup-wsl](https://github.com/vampire/setup-wsl) from 4.1.1 to 5.0.0. - [Release notes](https://github.com/vampire/setup-wsl/releases) - [Commits](https://github.com/vampire/setup-wsl/compare/v4.1.1...v5.0.0) --- updated-dependencies: - dependency-name: Vampire/setup-wsl dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4850f252c..039699af5 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -47,7 +47,7 @@ jobs: - name: Set up WSL (Windows) if: matrix.os-type == 'windows' - uses: Vampire/setup-wsl@v4.1.1 + uses: Vampire/setup-wsl@v5.0.0 with: distribution: Alpine additional-packages: bash From a41a0de46d2fdbb34207161bb0c180bd72958c8b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 6 Mar 2025 19:06:46 -0500 Subject: [PATCH 1125/1392] Use WSL1 on CI This avoids an occasional HTTP 403 error updating WSL for WSL2. For details on that issue and possible approaches, see: https://github.com/gitpython-developers/GitPython/pull/2008#pullrequestreview-2665805369 --- .github/workflows/pythonpackage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 039699af5..1a0210723 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -49,6 +49,7 @@ jobs: if: matrix.os-type == 'windows' uses: Vampire/setup-wsl@v5.0.0 with: + wsl-version: 1 distribution: Alpine additional-packages: bash From c13d998c03ec5268b5b6c361fe0c65854041b684 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 6 Mar 2025 19:45:41 -0500 Subject: [PATCH 1126/1392] Test on free-threaded Python See #2005. Right now, this does not limit by operating system, but that is just to verify that there are no OS-specific 3.13t problems we should know about right now; once that is verified, the macOS and Windows jobs will be removed (excluded) for the time being. The 3.13t jobs added here use `Quansight-Labs/setup-python`, not `actions/setup-python`. The latter also has the ability to use 3.13t since https://github.com/actions/python-versions/pull/319 and https://github.com/actions/setup-python/pull/973 (see also https://github.com/actions/setup-python/issues/771), but no version tag includes this feature yet. It can be used by using `@main` or `@...` where `...` is an OID. The former would risk pulling in other untested features we're no trying to test with, while the latter would not be easy to upgrade automatically as what we have now (we would be deliberately keeping a hash not at any tag that is already not the latest hash on any branch). In contrast, the `Quansight-Labs/setup-python` fork adds this feature while staying up to date with others. When `actions/setup-python` has a release (stable or prerelease) with this feature, we can switch to it. This could probably be done with less code duplication by using a matrix variable for the action to use. Instead, the "Set up Python" step is split in two, with opposite `if` conditions, so that each is capable of being recognized and upgraded by Dependabot if a new major version is released (in case this ends up remaining in place longer than expected). --- .github/workflows/pythonpackage.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 039699af5..b24c5cc57 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os-type: [ubuntu, macos, windows] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] exclude: - os-type: macos python-version: "3.7" # Not available for the ARM-based macOS runners. @@ -40,11 +40,20 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} + if: |- + !endsWith(matrix.python-version, 't') uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} + - name: Set up Python ${{ matrix.python-version }} (free-threaded) + if: endsWith(matrix.python-version, 't') + uses: Quansight-Labs/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: ${{ matrix.experimental }} + - name: Set up WSL (Windows) if: matrix.os-type == 'windows' uses: Vampire/setup-wsl@v5.0.0 From 56038c3e0382d87ccdb66d53964f038314c157fd Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 6 Mar 2025 22:39:23 -0500 Subject: [PATCH 1127/1392] Only test free-threaded Python on Linux For now, this omits macOS and Windows from the 3.13t ("threaded") tests. The plan in #2005 is to start without them, and no OS-specific problems have been identified so far. In particular, in the previous commit that adds 3.13t without excluding any operating systems, all tests in the macOS job passed as expected, and the Windows job had the same failure with the same message as in #1955, with no XFAIL changed to XPASS (which, if present, would suggest GC differences meriting further exploration of 3.13t on Windows). --- .github/workflows/pythonpackage.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index b24c5cc57..661df0693 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -17,8 +17,12 @@ jobs: exclude: - os-type: macos python-version: "3.7" # Not available for the ARM-based macOS runners. + - os-type: macos + python-version: "3.13t" - os-type: windows python-version: "3.13" # FIXME: Fix and enable Python 3.13 on Windows (#1955). + - os-type: windows + python-version: "3.13t" include: - os-ver: latest - os-type: ubuntu From 11f7fafada8348c8e8f699c7ab621be6d26b00a5 Mon Sep 17 00:00:00 2001 From: Kamil Kozik Date: Fri, 7 Mar 2025 18:39:06 +0100 Subject: [PATCH 1128/1392] `IndexFile._to_relative_path` - fix case where absolute path gets stripped of trailing slash --- git/index/base.py | 5 ++++- test/test_index.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/git/index/base.py b/git/index/base.py index 39cc9143c..65b1f9308 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -655,7 +655,10 @@ def _to_relative_path(self, path: PathLike) -> PathLike: raise InvalidGitRepositoryError("require non-bare repository") if not osp.normpath(str(path)).startswith(str(self.repo.working_tree_dir)): raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir)) - return os.path.relpath(path, self.repo.working_tree_dir) + result = os.path.relpath(path, self.repo.working_tree_dir) + if str(path).endswith(os.sep) and not result.endswith(os.sep): + result += os.sep + return result def _preprocess_add_items( self, items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]] diff --git a/test/test_index.py b/test/test_index.py index c586a0b5a..c42032e70 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -16,6 +16,7 @@ import subprocess import sys import tempfile +from unittest import mock from gitdb.base import IStream @@ -1015,6 +1016,27 @@ class Mocked: rel = index._to_relative_path(path) self.assertEqual(rel, os.path.relpath(path, root)) + def test__to_relative_path_absolute_trailing_slash(self): + repo_root = os.path.join(osp.abspath(os.sep), "directory1", "repo_root") + + class Mocked: + bare = False + git_dir = repo_root + working_tree_dir = repo_root + + repo = Mocked() + path = os.path.join(repo_root, f"directory2{os.sep}") + index = IndexFile(repo) + + expected_path = f"directory2{os.sep}" + actual_path = index._to_relative_path(path) + self.assertEqual(expected_path, actual_path) + + with mock.patch("git.index.base.os.path") as ospath_mock: + ospath_mock.relpath.return_value = f"directory2{os.sep}" + actual_path = index._to_relative_path(path) + self.assertEqual(expected_path, actual_path) + @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Absent, reason="Can't run a hook on Windows without bash.exe.", From 94151aa2ca9f16491a0cf2344b4daa8bf7b41d70 Mon Sep 17 00:00:00 2001 From: Andrej730 Date: Wed, 19 Mar 2025 13:17:46 +0500 Subject: [PATCH 1129/1392] Use property decorator to support typing --- git/refs/symbolic.py | 41 ++++++++++++++++++++++++----------------- git/repo/base.py | 40 +++++++++++++++++++++------------------- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 510850b2e..1b90a3115 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -39,7 +39,6 @@ if TYPE_CHECKING: from git.config import GitConfigParser from git.objects.commit import Actor - from git.refs import Head, TagReference, RemoteReference, Reference from git.refs.log import RefLogEntry from git.repo import Repo @@ -387,17 +386,23 @@ def set_object( # set the commit on our reference return self._get_reference().set_object(object, logmsg) - commit = property( - _get_commit, - set_commit, # type: ignore[arg-type] - doc="Query or set commits directly", - ) + @property + def commit(self) -> "Commit": + """Query or set commits directly""" + return self._get_commit() + + @commit.setter + def commit(self, commit: Union[Commit, "SymbolicReference", str]) -> "SymbolicReference": + return self.set_commit(commit) + + @property + def object(self) -> AnyGitObject: + """Return the object our ref currently refers to""" + return self._get_object() - object = property( - _get_object, - set_object, # type: ignore[arg-type] - doc="Return the object our ref currently refers to", - ) + @object.setter + def object(self, object: Union[AnyGitObject, "SymbolicReference", str]) -> "SymbolicReference": + return self.set_object(object) def _get_reference(self) -> "SymbolicReference": """ @@ -496,12 +501,14 @@ def set_reference( return self # Aliased reference - reference: Union["Head", "TagReference", "RemoteReference", "Reference"] - reference = property( # type: ignore[assignment] - _get_reference, - set_reference, # type: ignore[arg-type] - doc="Returns the Reference we point to", - ) + @property + def reference(self) -> "SymbolicReference": + return self._get_reference() + + @reference.setter + def reference(self, ref: Union[AnyGitObject, "SymbolicReference", str]) -> "SymbolicReference": + return self.set_reference(ref) + ref = reference def is_valid(self) -> bool: diff --git a/git/repo/base.py b/git/repo/base.py index db89cdf41..cbf54f222 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -354,21 +354,19 @@ def __ne__(self, rhs: object) -> bool: def __hash__(self) -> int: return hash(self.git_dir) - # Description property - def _get_description(self) -> str: + @property + def description(self) -> str: + """The project's description""" filename = osp.join(self.git_dir, "description") with open(filename, "rb") as fp: return fp.read().rstrip().decode(defenc) - def _set_description(self, descr: str) -> None: + @description.setter + def description(self, descr: str) -> None: filename = osp.join(self.git_dir, "description") with open(filename, "wb") as fp: fp.write((descr + "\n").encode(defenc)) - description = property(_get_description, _set_description, doc="the project's description") - del _get_description - del _set_description - @property def working_tree_dir(self) -> Optional[PathLike]: """ @@ -885,13 +883,14 @@ def _set_daemon_export(self, value: object) -> None: elif not value and fileexists: os.unlink(filename) - daemon_export = property( - _get_daemon_export, - _set_daemon_export, - doc="If True, git-daemon may export this repository", - ) - del _get_daemon_export - del _set_daemon_export + @property + def daemon_export(self) -> bool: + """If True, git-daemon may export this repository""" + return self._get_daemon_export() + + @daemon_export.setter + def daemon_export(self, value: object) -> None: + self._set_daemon_export(value) def _get_alternates(self) -> List[str]: """The list of alternates for this repo from which objects can be retrieved. @@ -929,11 +928,14 @@ def _set_alternates(self, alts: List[str]) -> None: with open(alternates_path, "wb") as f: f.write("\n".join(alts).encode(defenc)) - alternates = property( - _get_alternates, - _set_alternates, - doc="Retrieve a list of alternates paths or set a list paths to be used as alternates", - ) + @property + def alternates(self) -> List[str]: + """Retrieve a list of alternates paths or set a list paths to be used as alternates""" + return self._get_alternates() + + @alternates.setter + def alternates(self, alts: List[str]) -> None: + self._set_alternates(alts) def is_dirty( self, From 3d979a6c307b2a03f08d4bdbc7fb3716d8c17c94 Mon Sep 17 00:00:00 2001 From: Yuki Kobayashi Date: Tue, 18 Mar 2025 08:22:04 +0000 Subject: [PATCH 1130/1392] Fix some incorrect sphinx markups in the docstrings Fixed some markups so [the api reference](https://gitpython.readthedocs.io/en/stable/reference.html) is rendered correctly. --- git/index/base.py | 10 +++++----- git/objects/base.py | 4 ++-- git/objects/commit.py | 4 ++-- git/repo/base.py | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 65b1f9308..a95762dca 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -508,7 +508,7 @@ def iter_blobs( :param predicate: Function(t) returning ``True`` if tuple(stage, Blob) should be yielded by - the iterator. A default filter, the `~git.index.typ.BlobFilter`, allows you + the iterator. A default filter, the :class:`~git.index.typ.BlobFilter`, allows you to yield blobs only if they match a given list of paths. """ for entry in self.entries.values(): @@ -770,7 +770,7 @@ def add( - path string Strings denote a relative or absolute path into the repository pointing - to an existing file, e.g., ``CHANGES``, `lib/myfile.ext``, + to an existing file, e.g., ``CHANGES``, ``lib/myfile.ext``, ``/home/gitrepo/lib/myfile.ext``. Absolute paths must start with working tree directory of this index's @@ -789,7 +789,7 @@ def add( They are added at stage 0. - - :class:~`git.objects.blob.Blob` or + - :class:`~git.objects.blob.Blob` or :class:`~git.objects.submodule.base.Submodule` object Blobs are added as they are assuming a valid mode is set. @@ -815,7 +815,7 @@ def add( - :class:`~git.index.typ.BaseIndexEntry` or type - Handling equals the one of :class:~`git.objects.blob.Blob` objects, but + Handling equals the one of :class:`~git.objects.blob.Blob` objects, but the stage may be explicitly set. Please note that Index Entries require binary sha's. @@ -998,7 +998,7 @@ def remove( The path string may include globs, such as ``*.c``. - - :class:~`git.objects.blob.Blob` object + - :class:`~git.objects.blob.Blob` object Only the path portion is used in this case. diff --git a/git/objects/base.py b/git/objects/base.py index eeaebc09b..faf600c6b 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -122,7 +122,7 @@ def new(cls, repo: "Repo", id: Union[str, "Reference"]) -> AnyGitObject: :return: New :class:`Object` instance of a type appropriate to the object type behind `id`. The id of the newly created object will be a binsha even though the - input id may have been a `~git.refs.reference.Reference` or rev-spec. + input id may have been a :class:`~git.refs.reference.Reference` or rev-spec. :param id: :class:`~git.refs.reference.Reference`, rev-spec, or hexsha. @@ -218,7 +218,7 @@ class IndexObject(Object): """Base for all objects that can be part of the index file. The classes representing git object types that can be part of the index file are - :class:`~git.objects.tree.Tree and :class:`~git.objects.blob.Blob`. In addition, + :class:`~git.objects.tree.Tree` and :class:`~git.objects.blob.Blob`. In addition, :class:`~git.objects.submodule.base.Submodule`, which is not really a git object type but can be part of an index file, is also a subclass. """ diff --git a/git/objects/commit.py b/git/objects/commit.py index 0ceb46609..fbe0ee9c0 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -289,7 +289,7 @@ def name_rev(self) -> str: """ :return: String describing the commits hex sha based on the closest - `~git.refs.reference.Reference`. + :class:`~git.refs.reference.Reference`. :note: Mostly useful for UI purposes. @@ -349,7 +349,7 @@ def iter_items( return cls._iter_from_process_or_stream(repo, proc) def iter_parents(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) -> Iterator["Commit"]: - R"""Iterate _all_ parents of this commit. + R"""Iterate *all* parents of this commit. :param paths: Optional path or list of paths limiting the :class:`Commit`\s to those that diff --git a/git/repo/base.py b/git/repo/base.py index cbf54f222..7e918df8c 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -512,7 +512,7 @@ def create_submodule(self, *args: Any, **kwargs: Any) -> Submodule: def iter_submodules(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: """An iterator yielding Submodule instances. - See the `~git.objects.util.Traversable` interface for a description of `args` + See the :class:`~git.objects.util.Traversable` interface for a description of `args` and `kwargs`. :return: From 1abb399b90994f61b81c1aa4be608a85b07b73c7 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Tue, 25 Mar 2025 09:20:48 -0600 Subject: [PATCH 1131/1392] replace quansight-labs/setup-python with actions/setup-python --- .github/workflows/pythonpackage.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 5bedb6107..61088237d 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -44,20 +44,11 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - if: |- - !endsWith(matrix.python-version, 't') uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} - - name: Set up Python ${{ matrix.python-version }} (free-threaded) - if: endsWith(matrix.python-version, 't') - uses: Quansight-Labs/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: ${{ matrix.experimental }} - - name: Set up WSL (Windows) if: matrix.os-type == 'windows' uses: Vampire/setup-wsl@v5.0.0 From f2483a6151a99b7326b657fd945ec75f891e6462 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 May 2025 13:53:04 +0000 Subject: [PATCH 1132/1392] Bump Vampire/setup-wsl from 5.0.0 to 5.0.1 Bumps [Vampire/setup-wsl](https://github.com/vampire/setup-wsl) from 5.0.0 to 5.0.1. - [Release notes](https://github.com/vampire/setup-wsl/releases) - [Commits](https://github.com/vampire/setup-wsl/compare/v5.0.0...v5.0.1) --- updated-dependencies: - dependency-name: Vampire/setup-wsl dependency-version: 5.0.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 61088237d..9fd660c6b 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -51,7 +51,7 @@ jobs: - name: Set up WSL (Windows) if: matrix.os-type == 'windows' - uses: Vampire/setup-wsl@v5.0.0 + uses: Vampire/setup-wsl@v5.0.1 with: wsl-version: 1 distribution: Alpine From ec0087227ed37cf1df441e4f0f73daa25895c4b5 Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Wed, 21 May 2025 15:48:34 -0400 Subject: [PATCH 1133/1392] add tests for is_cygwin_git --- test/test_util.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/test_util.py b/test/test_util.py index dad2f3dcd..ed57dcbae 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -34,6 +34,7 @@ LockFile, cygpath, decygpath, + is_cygwin_git, get_user_id, remove_password_if_present, rmtree, @@ -349,6 +350,24 @@ def test_decygpath(self, wpath, cpath): assert wcpath == wpath.replace("/", "\\"), cpath +class TestIsCygwinGit: + """Tests for :func:`is_cygwin_git`""" + + def test_on_path_executable(self): + match sys.platform: + case "cygwin": + assert is_cygwin_git("git") + case _: + assert not is_cygwin_git("git") + + def test_none_executable(self): + assert not is_cygwin_git(None) + + def test_with_missing_uname(self): + """Test for handling when `uname` isn't in the same directory as `git`""" + assert not is_cygwin_git("/bogus_path/git") + + class _Member: """A member of an IterableList.""" From de5e57caf26e5f6772ac02a3944e58e0aba177f3 Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Wed, 21 May 2025 15:49:07 -0400 Subject: [PATCH 1134/1392] check for the existence/execute bit on the uname command before trying to run it --- git/util.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/git/util.py b/git/util.py index 9e8ac821d..f1a94c469 100644 --- a/git/util.py +++ b/git/util.py @@ -463,7 +463,13 @@ def _is_cygwin_git(git_executable: str) -> bool: git_dir = osp.dirname(res[0]) if res else "" # Just a name given, not a real path. + # Let's see if the same path has uname uname_cmd = osp.join(git_dir, "uname") + if not (osp.isfile(uname_cmd) and os.access(uname_cmd, os.X_OK)): + _logger.debug(f"File {uname_cmd} either does not exist or is not executable.") + _is_cygwin_cache[git_executable] = is_cygwin + return is_cygwin + process = subprocess.Popen([uname_cmd], stdout=subprocess.PIPE, universal_newlines=True) uname_out, _ = process.communicate() # retcode = process.poll() From 428be1a504e2b30c195786d5ca6708f4c39d90a7 Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Wed, 21 May 2025 15:51:29 -0400 Subject: [PATCH 1135/1392] adding self to authors --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 45b14c961..b57113edd 100644 --- a/AUTHORS +++ b/AUTHORS @@ -55,5 +55,6 @@ Contributors are: -Eliah Kagan -Ethan Lin -Jonas Scharpf +-Gordon Marx Portions derived from other open source works and are clearly marked. From 58ff723e6757af3c508e2a9424d287e5016ac230 Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Thu, 22 May 2025 08:54:42 -0400 Subject: [PATCH 1136/1392] Revert "check for the existence/execute bit on the uname command before trying to run it" This reverts commit de5e57caf26e5f6772ac02a3944e58e0aba177f3. --- git/util.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/git/util.py b/git/util.py index f1a94c469..9e8ac821d 100644 --- a/git/util.py +++ b/git/util.py @@ -463,13 +463,7 @@ def _is_cygwin_git(git_executable: str) -> bool: git_dir = osp.dirname(res[0]) if res else "" # Just a name given, not a real path. - # Let's see if the same path has uname uname_cmd = osp.join(git_dir, "uname") - if not (osp.isfile(uname_cmd) and os.access(uname_cmd, os.X_OK)): - _logger.debug(f"File {uname_cmd} either does not exist or is not executable.") - _is_cygwin_cache[git_executable] = is_cygwin - return is_cygwin - process = subprocess.Popen([uname_cmd], stdout=subprocess.PIPE, universal_newlines=True) uname_out, _ = process.communicate() # retcode = process.poll() From 71879840b8a72890f63fcadf7a92694106610ef0 Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Thu, 22 May 2025 09:05:13 -0400 Subject: [PATCH 1137/1392] use pathlib.Path instead of os.path.isfile --- git/util.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/git/util.py b/git/util.py index 9e8ac821d..3b6917fc4 100644 --- a/git/util.py +++ b/git/util.py @@ -464,6 +464,12 @@ def _is_cygwin_git(git_executable: str) -> bool: # Just a name given, not a real path. uname_cmd = osp.join(git_dir, "uname") + + if not (pathlib.Path(uname_cmd).exists() and os.access(uname_cmd, os.X_OK)): + _logger.debug(f"Failed checking if running in CYGWIN: {uname_cmd} is not an executable") + _is_cygwin_cache[git_executable] = is_cygwin + return is_cygwin + process = subprocess.Popen([uname_cmd], stdout=subprocess.PIPE, universal_newlines=True) uname_out, _ = process.communicate() # retcode = process.poll() From 3f5a942be8293bb8c663631756885f877afc22e7 Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Thu, 22 May 2025 09:06:02 -0400 Subject: [PATCH 1138/1392] don't keep checking if sys.platform isn't 'cygwin' --- git/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index 3b6917fc4..989f9196c 100644 --- a/git/util.py +++ b/git/util.py @@ -490,7 +490,7 @@ def is_cygwin_git(git_executable: PathLike) -> bool: ... def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: - if sys.platform == "win32": # TODO: See if we can use `sys.platform != "cygwin"`. + if sys.platform != "cygwin": return False elif git_executable is None: return False From 226f4ffcaf2deed4598b9ff53902a12f6483c069 Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Thu, 22 May 2025 09:15:50 -0400 Subject: [PATCH 1139/1392] check that uname_cmd points to a file; if uname_cmd were a directory, it could both exist and have os.X_OK but not work --- git/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index 989f9196c..32a808c60 100644 --- a/git/util.py +++ b/git/util.py @@ -465,7 +465,7 @@ def _is_cygwin_git(git_executable: str) -> bool: # Just a name given, not a real path. uname_cmd = osp.join(git_dir, "uname") - if not (pathlib.Path(uname_cmd).exists() and os.access(uname_cmd, os.X_OK)): + if not (pathlib.Path(uname_cmd).isfile() and os.access(uname_cmd, os.X_OK)): _logger.debug(f"Failed checking if running in CYGWIN: {uname_cmd} is not an executable") _is_cygwin_cache[git_executable] = is_cygwin return is_cygwin From 22d6284b099e99716da101198ce113c96e35423a Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Thu, 22 May 2025 09:24:37 -0400 Subject: [PATCH 1140/1392] don't use match-case, since it's a 3.10 feature --- test/test_util.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index ed57dcbae..de80ca3af 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -354,11 +354,10 @@ class TestIsCygwinGit: """Tests for :func:`is_cygwin_git`""" def test_on_path_executable(self): - match sys.platform: - case "cygwin": - assert is_cygwin_git("git") - case _: - assert not is_cygwin_git("git") + if sys.platform == "cygwin": + assert is_cygwin_git("git") + else: + assert not is_cygwin_git("git") def test_none_executable(self): assert not is_cygwin_git(None) From b1289eeb3676e63f4261518655ee90808e9d3d1f Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Thu, 22 May 2025 10:56:26 -0400 Subject: [PATCH 1141/1392] it's is_file(), not isfile() --- git/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index 32a808c60..4071fdc14 100644 --- a/git/util.py +++ b/git/util.py @@ -465,7 +465,7 @@ def _is_cygwin_git(git_executable: str) -> bool: # Just a name given, not a real path. uname_cmd = osp.join(git_dir, "uname") - if not (pathlib.Path(uname_cmd).isfile() and os.access(uname_cmd, os.X_OK)): + if not (pathlib.Path(uname_cmd).is_file() and os.access(uname_cmd, os.X_OK)): _logger.debug(f"Failed checking if running in CYGWIN: {uname_cmd} is not an executable") _is_cygwin_cache[git_executable] = is_cygwin return is_cygwin @@ -490,6 +490,7 @@ def is_cygwin_git(git_executable: PathLike) -> bool: ... def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: + _logger.debug(f"{sys.platform=}, {git_executable=}") if sys.platform != "cygwin": return False elif git_executable is None: From cffa264bdc1dd09a13a85cc27417b85628273e14 Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Thu, 22 May 2025 11:11:48 -0400 Subject: [PATCH 1142/1392] turns out f-strings before 3.8 don't support {variable=} notation, take that out --- git/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index 4071fdc14..66175ea33 100644 --- a/git/util.py +++ b/git/util.py @@ -490,7 +490,7 @@ def is_cygwin_git(git_executable: PathLike) -> bool: ... def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: - _logger.debug(f"{sys.platform=}, {git_executable=}") + _logger.debug(f"sys.platform = {sys.platform}, git_executable = {git_executable}") if sys.platform != "cygwin": return False elif git_executable is None: From 8dc75521bd42db19d41e8d92ce5204dfe1a594ac Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Thu, 22 May 2025 15:48:54 -0400 Subject: [PATCH 1143/1392] remove type assertions from util.py --- git/util.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/git/util.py b/git/util.py index 9e8ac821d..0d09c3f09 100644 --- a/git/util.py +++ b/git/util.py @@ -1200,8 +1200,6 @@ def __getattr__(self, attr: str) -> T_IterableObj: return list.__getattribute__(self, attr) def __getitem__(self, index: Union[SupportsIndex, int, slice, str]) -> T_IterableObj: # type: ignore[override] - assert isinstance(index, (int, str, slice)), "Index of IterableList should be an int or str" - if isinstance(index, int): return list.__getitem__(self, index) elif isinstance(index, slice): @@ -1214,8 +1212,6 @@ def __getitem__(self, index: Union[SupportsIndex, int, slice, str]) -> T_Iterabl # END handle getattr def __delitem__(self, index: Union[SupportsIndex, int, slice, str]) -> None: - assert isinstance(index, (int, str)), "Index of IterableList should be an int or str" - delindex = cast(int, index) if not isinstance(index, int): delindex = -1 From 36a893bbd47a08df80e0266f6ce5182915511833 Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Fri, 23 May 2025 07:24:43 -0400 Subject: [PATCH 1144/1392] add self to AUTHORS, add Emacs tempfiles to .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index d85569405..eab294a65 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ __pycache__/ # Transient editor files *.swp *~ +\#*# +.#*# # Editor configuration nbproject From c441316b7714207eaf2c13be5104d15ec69943bd Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Tue, 27 May 2025 10:58:24 -0400 Subject: [PATCH 1145/1392] use `strings` instead of `uname` to detect cygwin --- git/util.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/git/util.py b/git/util.py index 66175ea33..53475eeb3 100644 --- a/git/util.py +++ b/git/util.py @@ -457,23 +457,24 @@ def _is_cygwin_git(git_executable: str) -> bool: if is_cygwin is None: is_cygwin = False try: - git_dir = osp.dirname(git_executable) + git_cmd = pathlib.Path(git_executable) + git_dir = git_cmd.parent + if not git_dir: res = py_where(git_executable) - git_dir = osp.dirname(res[0]) if res else "" + git_dir = pathlib.Path(res[0]).parent if res else "" - # Just a name given, not a real path. - uname_cmd = osp.join(git_dir, "uname") + # If it's a cygwin git, it'll have cygwin in the output of `strings git` + strings_cmd = pathlib.Path(git_dir, "strings") - if not (pathlib.Path(uname_cmd).is_file() and os.access(uname_cmd, os.X_OK)): - _logger.debug(f"Failed checking if running in CYGWIN: {uname_cmd} is not an executable") - _is_cygwin_cache[git_executable] = is_cygwin + if not (pathlib.Path(strings_cmd).is_file() and os.access(strings_cmd, os.X_OK)): + _logger.debug(f"Failed checking if running in CYGWIN: {strings_cmd} is not an executable") + _is_cygwin_cache[git_executable] = False return is_cygwin - process = subprocess.Popen([uname_cmd], stdout=subprocess.PIPE, universal_newlines=True) - uname_out, _ = process.communicate() - # retcode = process.poll() - is_cygwin = "CYGWIN" in uname_out + process = subprocess.Popen([strings_cmd, git_cmd], stdout=subprocess.PIPE, text=True) + strings_output, _ = process.communicate() + is_cygwin = any(x for x in strings_output if "cygwin" in x.lower()) except Exception as ex: _logger.debug("Failed checking if running in CYGWIN due to: %r", ex) _is_cygwin_cache[git_executable] = is_cygwin From 0df08181f1012f0e591f998dcc57fd594e754d98 Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Tue, 27 May 2025 11:32:23 -0400 Subject: [PATCH 1146/1392] debug printing --- git/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git/util.py b/git/util.py index 53475eeb3..81761f15c 100644 --- a/git/util.py +++ b/git/util.py @@ -475,6 +475,8 @@ def _is_cygwin_git(git_executable: str) -> bool: process = subprocess.Popen([strings_cmd, git_cmd], stdout=subprocess.PIPE, text=True) strings_output, _ = process.communicate() is_cygwin = any(x for x in strings_output if "cygwin" in x.lower()) + if not is_cygwin: + _logger.debug(f"is not cygwin: {strings_output}") except Exception as ex: _logger.debug("Failed checking if running in CYGWIN due to: %r", ex) _is_cygwin_cache[git_executable] = is_cygwin From 58f77105f5163408a9bf323518b96646ab413561 Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Wed, 28 May 2025 11:15:06 -0400 Subject: [PATCH 1147/1392] Revert "debug printing" This reverts commit 0df08181f1012f0e591f998dcc57fd594e754d98. --- git/util.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/git/util.py b/git/util.py index 81761f15c..53475eeb3 100644 --- a/git/util.py +++ b/git/util.py @@ -475,8 +475,6 @@ def _is_cygwin_git(git_executable: str) -> bool: process = subprocess.Popen([strings_cmd, git_cmd], stdout=subprocess.PIPE, text=True) strings_output, _ = process.communicate() is_cygwin = any(x for x in strings_output if "cygwin" in x.lower()) - if not is_cygwin: - _logger.debug(f"is not cygwin: {strings_output}") except Exception as ex: _logger.debug("Failed checking if running in CYGWIN due to: %r", ex) _is_cygwin_cache[git_executable] = is_cygwin From 1731c1e377aca056d1a7acf2dd6df0e68a4e9e7e Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Wed, 28 May 2025 11:15:23 -0400 Subject: [PATCH 1148/1392] Revert "use `strings` instead of `uname` to detect cygwin" This reverts commit c441316b7714207eaf2c13be5104d15ec69943bd. --- git/util.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/git/util.py b/git/util.py index 53475eeb3..66175ea33 100644 --- a/git/util.py +++ b/git/util.py @@ -457,24 +457,23 @@ def _is_cygwin_git(git_executable: str) -> bool: if is_cygwin is None: is_cygwin = False try: - git_cmd = pathlib.Path(git_executable) - git_dir = git_cmd.parent - + git_dir = osp.dirname(git_executable) if not git_dir: res = py_where(git_executable) - git_dir = pathlib.Path(res[0]).parent if res else "" + git_dir = osp.dirname(res[0]) if res else "" - # If it's a cygwin git, it'll have cygwin in the output of `strings git` - strings_cmd = pathlib.Path(git_dir, "strings") + # Just a name given, not a real path. + uname_cmd = osp.join(git_dir, "uname") - if not (pathlib.Path(strings_cmd).is_file() and os.access(strings_cmd, os.X_OK)): - _logger.debug(f"Failed checking if running in CYGWIN: {strings_cmd} is not an executable") - _is_cygwin_cache[git_executable] = False + if not (pathlib.Path(uname_cmd).is_file() and os.access(uname_cmd, os.X_OK)): + _logger.debug(f"Failed checking if running in CYGWIN: {uname_cmd} is not an executable") + _is_cygwin_cache[git_executable] = is_cygwin return is_cygwin - process = subprocess.Popen([strings_cmd, git_cmd], stdout=subprocess.PIPE, text=True) - strings_output, _ = process.communicate() - is_cygwin = any(x for x in strings_output if "cygwin" in x.lower()) + process = subprocess.Popen([uname_cmd], stdout=subprocess.PIPE, universal_newlines=True) + uname_out, _ = process.communicate() + # retcode = process.poll() + is_cygwin = "CYGWIN" in uname_out except Exception as ex: _logger.debug("Failed checking if running in CYGWIN due to: %r", ex) _is_cygwin_cache[git_executable] = is_cygwin From f3ab5d3810b1e8c1f0365d26eea3e91808d8e4af Mon Sep 17 00:00:00 2001 From: Gordon Marx Date: Wed, 28 May 2025 11:21:28 -0400 Subject: [PATCH 1149/1392] incorporate review feedback from @EliahKagan --- git/util.py | 3 ++- test/test_util.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/git/util.py b/git/util.py index 66175ea33..a10609185 100644 --- a/git/util.py +++ b/git/util.py @@ -490,7 +490,8 @@ def is_cygwin_git(git_executable: PathLike) -> bool: ... def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: - _logger.debug(f"sys.platform = {sys.platform}, git_executable = {git_executable}") + # TODO: when py3.7 support is dropped, use the new interpolation f"{variable=}" + _logger.debug(f"sys.platform={sys.platform!r}, git_executable={git_executable!r}") if sys.platform != "cygwin": return False elif git_executable is None: diff --git a/test/test_util.py b/test/test_util.py index de80ca3af..000830f41 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -354,6 +354,7 @@ class TestIsCygwinGit: """Tests for :func:`is_cygwin_git`""" def test_on_path_executable(self): + # Currently we assume tests run on Cygwin use Cygwin git. See #533 and #1455 for background. if sys.platform == "cygwin": assert is_cygwin_git("git") else: From b7ce712c631c0b59566af43e7a4b00cc1dbeba3b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 30 May 2025 12:36:37 -0400 Subject: [PATCH 1150/1392] Use newer ruff style This updates `ruff` in `.pre-commit-config.yaml` from 0.6.0 to 0.11.12, changes its `id` from the legacy `ruff` alias to `ruff-check` (which is better distinguished from `ruff-format`, which we also have a hook for), and applies the few style changes it newly recommends throughout the code. The style changes seem to make things slightly clearer overall. This also updates some other pre-commit hooks, but those don't require any changes to the code. Currently the `ruff` dependency in `requirements-dev.txt` doesn't specify a version, so no change is needed there. This update may be seen as bringing the `pre-commit` version in line with what users will usually have locally with `pip install -e ".[test]"`. The `pre-commit` hooks are how linting is currently done on CI, so this is updating `ruff` for CI. That's the most significant effect of this change. (`pre-commit` is run for linting on CI probably much more often than it is used locally, to manage pre-commit hooks or otherwise, in GitPython development.) --- .pre-commit-config.yaml | 10 +++++----- git/cmd.py | 4 ++-- git/refs/log.py | 2 +- git/repo/fun.py | 2 +- test/test_git.py | 2 +- test/test_quick_doc.py | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 424cc5f37..737b56d45 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,15 +1,15 @@ repos: - repo: https://github.com/codespell-project/codespell - rev: v2.3.0 + rev: v2.4.1 hooks: - id: codespell additional_dependencies: [tomli] exclude: ^test/fixtures/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.0 + rev: v0.11.12 hooks: - - id: ruff + - id: ruff-check args: ["--fix"] exclude: ^git/ext/ - id: ruff-format @@ -23,7 +23,7 @@ repos: exclude: ^test/fixtures/polyglot$|^git/ext/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: end-of-file-fixer exclude: ^test/fixtures/|COPYING|LICENSE @@ -33,6 +33,6 @@ repos: - id: check-merge-conflict - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.19 + rev: v0.24.1 hooks: - id: validate-pyproject diff --git a/git/cmd.py b/git/cmd.py index 2048a43fa..7f46edc8f 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -207,7 +207,7 @@ def pump_stream( ) if stderr_handler: error_str: Union[str, bytes] = ( - "error: process killed because it timed out." f" kill_after_timeout={kill_after_timeout} seconds" + f"error: process killed because it timed out. kill_after_timeout={kill_after_timeout} seconds" ) if not decode_streams and isinstance(p_stderr, BinaryIO): # Assume stderr_handler needs binary input. @@ -1319,7 +1319,7 @@ def communicate() -> Tuple[AnyStr, AnyStr]: out, err = proc.communicate() watchdog.cancel() if kill_check.is_set(): - err = 'Timeout: the command "%s" did not complete in %d ' "secs." % ( + err = 'Timeout: the command "%s" did not complete in %d secs.' % ( " ".join(redacted_command), timeout, ) diff --git a/git/refs/log.py b/git/refs/log.py index 17e3a94b3..8f2f2cd38 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -126,7 +126,7 @@ def from_line(cls, line: bytes) -> "RefLogEntry": elif len(fields) == 2: info, msg = fields else: - raise ValueError("Line must have up to two TAB-separated fields." " Got %s" % repr(line_str)) + raise ValueError("Line must have up to two TAB-separated fields. Got %s" % repr(line_str)) # END handle first split oldhexsha = info[:40] diff --git a/git/repo/fun.py b/git/repo/fun.py index 125ba5936..1c995c6c6 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -405,7 +405,7 @@ def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: # END end handle tag except (IndexError, AttributeError) as e: raise BadName( - f"Invalid revision spec '{rev}' - not enough " f"parent commits to reach '{token}{int(num)}'" + f"Invalid revision spec '{rev}' - not enough parent commits to reach '{token}{int(num)}'" ) from e # END exception handling # END parse loop diff --git a/test/test_git.py b/test/test_git.py index 274511f8d..5bcf89bdd 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -747,7 +747,7 @@ def test_environment(self, rw_dir): path = osp.join(rw_dir, "failing-script.sh") with open(path, "wt") as stream: - stream.write("#!/usr/bin/env sh\n" "echo FOO\n") + stream.write("#!/usr/bin/env sh\necho FOO\n") os.chmod(path, 0o777) rw_repo = Repo.init(osp.join(rw_dir, "repo")) diff --git a/test/test_quick_doc.py b/test/test_quick_doc.py index 4ef75f4aa..98658e02f 100644 --- a/test/test_quick_doc.py +++ b/test/test_quick_doc.py @@ -173,7 +173,7 @@ def test_cloned_repo_object(self, local_dir): # [15-test_cloned_repo_object] def print_files_from_git(root, level=0): for entry in root: - print(f'{"-" * 4 * level}| {entry.path}, {entry.type}') + print(f"{'-' * 4 * level}| {entry.path}, {entry.type}") if entry.type == "tree": print_files_from_git(entry, level + 1) From 0c0fc7ee97c14088ef1705d756e04dadd9cfdd8a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 30 May 2025 14:37:03 -0400 Subject: [PATCH 1151/1392] Temporarily remove CodeQL workflow file So GitHub can regenerate a fresh new one based on current defaults. --- .github/workflows/codeql.yml | 80 ------------------------------------ 1 file changed, 80 deletions(-) delete mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index ae5241898..000000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,80 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - pull_request: - schedule: - - cron: '27 10 * * 3' - -jobs: - analyze: - name: Analyze - # Runner size impacts CodeQL analysis time. To learn more, please see: - # - https://gh.io/recommended-hardware-resources-for-running-codeql - # - https://gh.io/supported-runners-and-hardware-resources - # - https://gh.io/using-larger-runners - # Consider using larger runners for possible analysis time improvements. - runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} - timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'python' ] - # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] - # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both - # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.language }} - setup-python-dependencies: false - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v3 - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 - with: - category: "/language:${{matrix.language}}" From 847ead6948efa5d33a8798395220ed6b719a56f2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 30 May 2025 14:40:38 -0400 Subject: [PATCH 1152/1392] Recreate `codeq.yml` with current defaults This adds CodeQL scanning of GitHub Actions, while continuing to scan Python as well. This will subsequently be customized slightly to restore some elements of the preivous custom workflow that we may prefer. --- .github/workflows/codeql.yml | 100 +++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..450166503 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,100 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL Advanced" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '36 18 * * 0' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: actions + build-mode: none + - language: python + build-mode: none + # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Add any setup steps before running the `github/codeql-action/init` action. + # This includes steps like installing compilers or runtimes (`actions/setup-node` + # or others). This is typically only required for manual builds. + # - name: Setup runtime (example) + # uses: actions/setup-example@v1 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" From d7ce09fa75a7a86ed089ecff330ee86333df5200 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 30 May 2025 14:51:21 -0400 Subject: [PATCH 1153/1392] Keep running CodeQL on all branches, etc. This restores three aspects of the previous `codeql.yml`: - Run it on all branches, not just `main`. - Run it on the previous schedule rather than the new one, since there's no reason to change the schedule (though there's no reason to be attached to the old schedule either). - Use "CodeQL" rather than "CodeQL Advanced" as the workflow `name`, since this takes up less horizontal space when reading the reports from the checks. Of these, only the first is really significant. --- .github/workflows/codeql.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 450166503..3d273f9c7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -9,15 +9,13 @@ # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # -name: "CodeQL Advanced" +name: "CodeQL" on: push: - branches: [ "main" ] pull_request: - branches: [ "main" ] schedule: - - cron: '36 18 * * 0' + - cron: '27 10 * * 3' jobs: analyze: From 89dbd4a85ae78c272e5714e827f70783df04f924 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 30 May 2025 14:58:09 -0400 Subject: [PATCH 1154/1392] Remove unnecessary permissions from `codeql.yml` This is another change back to the way we had it before, but the removals are based specifically on the guidance in the default workflow comments about why each permission was given by default. --- .github/workflows/codeql.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 3d273f9c7..2bee952af 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -27,16 +27,8 @@ jobs: # Consider using larger runners or machines with greater resources for possible analysis time improvements. runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} permissions: - # required for all workflows security-events: write - # required to fetch internal or private CodeQL packs - packages: read - - # only required for workflows in private repositories - actions: read - contents: read - strategy: fail-fast: false matrix: From a9833d635dd5201cd94cc9d061590e41e24ea0cc Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 30 May 2025 15:38:00 -0400 Subject: [PATCH 1155/1392] Specify explicit `contents: read` workflow permissions Three CI workflows that need only `contents: read` permissions and no other permissions did not have explicit permissions set, and would therefore be given default permissions configured for the repository, which might be more expansive than the workflows need. It is recommended to set explicit workflow permissions [1]. This does that, specifying permissions as `pythonpackage.yml` already did, and closing three `actions/missing-workflow-permissions` CodeQL alerts (new since #2032 enabled scanning of GHA workflows). [1]: https://codeql.github.com/codeql-query-help/actions/actions-missing-workflow-permissions/ --- .github/workflows/alpine-test.yml | 3 +++ .github/workflows/cygwin-test.yml | 3 +++ .github/workflows/lint.yml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index bd09a939b..513c65bb8 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -2,6 +2,9 @@ name: test-alpine on: [push, pull_request, workflow_dispatch] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 278777907..572a9197e 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -2,6 +2,9 @@ name: test-cygwin on: [push, pull_request, workflow_dispatch] +permissions: + contents: read + jobs: test: runs-on: windows-latest diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a0e81a993..ceba0dd85 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,6 +2,9 @@ name: Lint on: [push, pull_request, workflow_dispatch] +permissions: + contents: read + jobs: lint: runs-on: ubuntu-latest From 0664fe8e5f05dc13f733d91368d6d42db6852b63 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 30 May 2025 16:21:55 -0400 Subject: [PATCH 1156/1392] Specify explicit `contents: read` workflow permissions This change is analogous to gitpython-developers/GitPython#2033. See also gitpython-developers/gitdb#121. --- .github/workflows/pythonpackage.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index ab0a8ef19..1671d9ade 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -5,6 +5,9 @@ name: Python package on: [push, pull_request, workflow_dispatch] +permissions: + contents: read + jobs: build: strategy: From d7a7b3b1d398b3c70997b2971769560ff6bf7491 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 30 May 2025 16:18:10 -0400 Subject: [PATCH 1157/1392] Specify explicit `contents: read` workflow permissions This change is analogous to gitpython-developers/GitPython#2033. See also gitpython-developers/smmap#60. --- .github/workflows/pythonpackage.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 907698d0e..8fd6369a1 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -5,6 +5,9 @@ name: Python package on: [push, pull_request, workflow_dispatch] +permissions: + contents: read + jobs: build: From 8d57ac71980d7fc688acbdd8a45e1f7e0023bc81 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 30 May 2025 16:34:24 -0400 Subject: [PATCH 1158/1392] Add CI test job for no-GIL ("threaded") Python 3.13 See https://github.com/gitpython-developers/GitPython/issues/2005. The rationale is that, while this is probably less important to do in gitdb and smmap, any failure that arises for this in GitPython would likely raise the question of whether a correspond problem has begun to occur in gitdb and smmap. (Both gitdb and smmap provide helpers used in GitPython even when the in-memory object database is not used, and failures may plausibly occur for reasons other than code changes because of the finicky nature of concurrency bugs and the potential for interactions affected by the runner image.) --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 8fd6369a1..c5d7e2b8c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] os: [ubuntu-latest] experimental: [false] include: From 90738a9de6bcc9c8a406aeee69201aff5b85c05a Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 30 May 2025 16:40:46 -0400 Subject: [PATCH 1159/1392] Add CI test job for no-GIL ("threaded") Python 3.13 See https://github.com/gitpython-developers/GitPython/issues/2005, and https://github.com/gitpython-developers/gitdb/pull/122 for rationale. In short, if the corresponding check starts to fail in GitPython, it may be useful to observe if there is a failure here as well. --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 1671d9ade..8935881e7 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -12,7 +12,7 @@ jobs: build: strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] include: - experimental: false - os: ubuntu-latest From 18b437b65b339f0d76a3c07f4cef1de4fbcb527a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jun 2025 11:11:28 +0000 Subject: [PATCH 1160/1392] Bump gitdb/ext/smmap from `8f82e6c` to `c6b53d3` Bumps [gitdb/ext/smmap](https://github.com/gitpython-developers/smmap) from `8f82e6c` to `c6b53d3`. - [Release notes](https://github.com/gitpython-developers/smmap/releases) - [Commits](https://github.com/gitpython-developers/smmap/compare/8f82e6c19661f9b735cc55cc89031a189e408894...c6b53d35deb82a38d5d07ca7712c1334a7a10c10) --- updated-dependencies: - dependency-name: gitdb/ext/smmap dependency-version: c6b53d35deb82a38d5d07ca7712c1334a7a10c10 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- gitdb/ext/smmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 8f82e6c19..c6b53d35d 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 8f82e6c19661f9b735cc55cc89031a189e408894 +Subproject commit c6b53d35deb82a38d5d07ca7712c1334a7a10c10 From 00b46127e54c104ac86333150708acdccce98cb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Jun 2025 03:22:44 +0000 Subject: [PATCH 1161/1392] Bump git/ext/gitdb from `f36c0cc` to `7e02fbd` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `f36c0cc` to `7e02fbd`. - [Release notes](https://github.com/gitpython-developers/gitdb/releases) - [Commits](https://github.com/gitpython-developers/gitdb/compare/f36c0cc42ea2f529291e441073f74e920988d4d2...7e02fbde5fcfcd52f541995fbcde22e85535adef) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-version: 7e02fbde5fcfcd52f541995fbcde22e85535adef dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index f36c0cc42..335c0f661 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit f36c0cc42ea2f529291e441073f74e920988d4d2 +Subproject commit 335c0f66173eecdc7b2597c2b6c3d1fde795df30 From 67468a2396f56f792ac4c139f43c98745e93a2b5 Mon Sep 17 00:00:00 2001 From: betaboon Date: Fri, 6 Jun 2025 22:41:28 +0200 Subject: [PATCH 1162/1392] Fix GitConfigParser not removing quotes from values --- git/config.py | 2 ++ test/fixtures/git_config_with_quotes | 3 +++ test/test_config.py | 8 +++++++- 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/git_config_with_quotes diff --git a/git/config.py b/git/config.py index de3508360..2df99a753 100644 --- a/git/config.py +++ b/git/config.py @@ -508,6 +508,8 @@ def string_decode(v: str) -> str: if len(optval) > 1 and optval[0] == '"' and optval[-1] != '"': is_multi_line = True optval = string_decode(optval[1:]) + elif len(optval) > 1 and optval[0] == '"' and optval[-1] == '"': + optval = optval[1:-1].strip() # END handle multi-line # Preserves multiple values for duplicate optnames. cursect.add(optname, optval) diff --git a/test/fixtures/git_config_with_quotes b/test/fixtures/git_config_with_quotes new file mode 100644 index 000000000..40e6710d9 --- /dev/null +++ b/test/fixtures/git_config_with_quotes @@ -0,0 +1,3 @@ +[user] + name = "Cody Veal" + email = "cveal05@gmail.com" diff --git a/test/test_config.py b/test/test_config.py index 92997422d..886d5b136 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -391,7 +391,7 @@ def test_complex_aliases(self): with GitConfigParser(file_obj, read_only=False) as w_config: self.assertEqual( w_config.get("alias", "rbi"), - '"!g() { git rebase -i origin/${1:-master} ; } ; g"', + "!g() { git rebase -i origin/${1:-master} ; } ; g", ) self.assertEqual( file_obj.getvalue(), @@ -406,6 +406,12 @@ def test_empty_config_value(self): with self.assertRaises(cp.NoOptionError): cr.get_value("color", "ui") + def test_config_with_quotes(self): + cr = GitConfigParser(fixture_path("git_config_with_quotes"), read_only=True) + + self.assertEqual(cr.get("user", "name"), "Cody Veal") + self.assertEqual(cr.get("user", "email"), "cveal05@gmail.com") + def test_get_values_works_without_requiring_any_other_calls_first(self): file_obj = self._to_memcache(fixture_path("git_config_multiple")) cr = GitConfigParser(file_obj, read_only=True) From 5f303202ee0666f5298ff39a5a129162b69ff790 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 01:08:07 -0400 Subject: [PATCH 1163/1392] Test a quoted config var with meaningful edge whitespace #2035 fixed issue #1923 by removing separate double quotation marks appearing on a single-line configuration variable when parsing a configuration file. However, it also stripped leading and trailing whitespace from the string obtained by removing the quotes. This adds a test case of a plausible scenario where such whitespace needs to be preserved and where a user would almost certainly expect it to preserve: setting a value like `# ` for `core.commentString`, in order to be able to easily create commit messages like this one, that contain a line that begins with a literal `#`, while still letting `#` in the more common case that it is followed by a space be interpreted as a comment. The effect of `git config --local core.commentString '# '` is to add a `commentString = "# "` line in the `[core]` section of `.git/config`. The changes in #2035 allow us to correctly parse more quoted strings than before, and almost allow us to parse this, but not quite, because of the `strip()` operation that turns `# ` into `#`. --- test/fixtures/git_config_with_quotes_whitespace | 2 ++ test/test_config.py | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 test/fixtures/git_config_with_quotes_whitespace diff --git a/test/fixtures/git_config_with_quotes_whitespace b/test/fixtures/git_config_with_quotes_whitespace new file mode 100644 index 000000000..c6014cc61 --- /dev/null +++ b/test/fixtures/git_config_with_quotes_whitespace @@ -0,0 +1,2 @@ +[core] + commentString = "# " diff --git a/test/test_config.py b/test/test_config.py index 886d5b136..671f34046 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -412,6 +412,10 @@ def test_config_with_quotes(self): self.assertEqual(cr.get("user", "name"), "Cody Veal") self.assertEqual(cr.get("user", "email"), "cveal05@gmail.com") + def test_config_with_quotes_with_literal_whitespace(self): + cr = GitConfigParser(fixture_path("git_config_with_quotes_whitespace"), read_only=True) + self.assertEqual(cr.get("core", "commentString"), "# ") + def test_get_values_works_without_requiring_any_other_calls_first(self): file_obj = self._to_memcache(fixture_path("git_config_multiple")) cr = GitConfigParser(file_obj, read_only=True) From bd2b930503ad5d97322aa81d7610ab743d915f84 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 00:13:46 -0400 Subject: [PATCH 1164/1392] Preserve quoted leading and trailing single-line whitespace At least in a single line, whitespace in a double-quoted value in a configuration file, like `name = " abc def "`, would presumably be intended. This removes the `strip()` call that is applied to text `ConfigParser` obtained by removing the double quotes around it. This slightly refines the changes in #2035 by dropping the `strip()` call while continuing to remove opening and closing double quotes. --- git/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/config.py b/git/config.py index 2df99a753..1d58db53a 100644 --- a/git/config.py +++ b/git/config.py @@ -509,7 +509,7 @@ def string_decode(v: str) -> str: is_multi_line = True optval = string_decode(optval[1:]) elif len(optval) > 1 and optval[0] == '"' and optval[-1] == '"': - optval = optval[1:-1].strip() + optval = optval[1:-1] # END handle multi-line # Preserves multiple values for duplicate optnames. cursect.add(optname, optval) From 5a8a4059d646fd313e81365ef240562556d8bd3f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 6 Jun 2025 23:16:45 -0400 Subject: [PATCH 1165/1392] Refactor Git.{AutoInterrupt,CatFileContentStream} nesting This makes `Git.AutoInterrupt` and `Git.CatFileContentStream` transparent aliases to top-level nonpublic `_AutoInterrupt` and `_CatFileContentStream` classes in the `cmd` module. This does not change the "public" interface. It also does not change metadata relevant to documentation: the `__name__` and `__qualname__` attributes are set explicitly to the values they had before when these classes were defined nested, so that Sphinx continues to document them (and to do so in full) in `Git` and as `Git.AutoInterrupt` and `Git.CatFileContentStream`. The purpose of this is to increase readability. The `Git` class is big and complex, with a number of long members and various forms of nesting. Since these two classes can be understood even without reading the code of the `Git` class, moving the definitions out of the `Git` class into top-level nonpublic classes will hopefully increase readability and help with maintenance. --- git/cmd.py | 440 +++++++++++++++++++++++++++-------------------------- 1 file changed, 226 insertions(+), 214 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 7f46edc8f..a6195880d 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -308,6 +308,230 @@ def dict_to_slots_and__excluded_are_none(self: object, d: Mapping[str, Any], exc ## -- End Utilities -- @} + +class _AutoInterrupt: + """Process wrapper that terminates the wrapped process on finalization. + + This kills/interrupts the stored process instance once this instance goes out of + scope. It is used to prevent processes piling up in case iterators stop reading. + + All attributes are wired through to the contained process object. + + The wait method is overridden to perform automatic status code checking and possibly + raise. + """ + + __slots__ = ("proc", "args", "status") + + # If this is non-zero it will override any status code during _terminate, used + # to prevent race conditions in testing. + _status_code_if_terminate: int = 0 + + def __init__(self, proc: Union[None, subprocess.Popen], args: Any) -> None: + self.proc = proc + self.args = args + self.status: Union[int, None] = None + + def _terminate(self) -> None: + """Terminate the underlying process.""" + if self.proc is None: + return + + proc = self.proc + self.proc = None + if proc.stdin: + proc.stdin.close() + if proc.stdout: + proc.stdout.close() + if proc.stderr: + proc.stderr.close() + # Did the process finish already so we have a return code? + try: + if proc.poll() is not None: + self.status = self._status_code_if_terminate or proc.poll() + return + except OSError as ex: + _logger.info("Ignored error after process had died: %r", ex) + + # It can be that nothing really exists anymore... + if os is None or getattr(os, "kill", None) is None: + return + + # Try to kill it. + try: + proc.terminate() + status = proc.wait() # Ensure the process goes away. + + self.status = self._status_code_if_terminate or status + except OSError as ex: + _logger.info("Ignored error after process had died: %r", ex) + # END exception handling + + def __del__(self) -> None: + self._terminate() + + def __getattr__(self, attr: str) -> Any: + return getattr(self.proc, attr) + + # TODO: Bad choice to mimic `proc.wait()` but with different args. + def wait(self, stderr: Union[None, str, bytes] = b"") -> int: + """Wait for the process and return its status code. + + :param stderr: + Previously read value of stderr, in case stderr is already closed. + + :warn: + May deadlock if output or error pipes are used and not handled separately. + + :raise git.exc.GitCommandError: + If the return status is not 0. + """ + if stderr is None: + stderr_b = b"" + stderr_b = force_bytes(data=stderr, encoding="utf-8") + status: Union[int, None] + if self.proc is not None: + status = self.proc.wait() + p_stderr = self.proc.stderr + else: # Assume the underlying proc was killed earlier or never existed. + status = self.status + p_stderr = None + + def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> bytes: + if stream: + try: + return stderr_b + force_bytes(stream.read()) + except (OSError, ValueError): + return stderr_b or b"" + else: + return stderr_b or b"" + + # END status handling + + if status != 0: + errstr = read_all_from_possibly_closed_stream(p_stderr) + _logger.debug("AutoInterrupt wait stderr: %r" % (errstr,)) + raise GitCommandError(remove_password_if_present(self.args), status, errstr) + return status + + +_AutoInterrupt.__name__ = "AutoInterrupt" +_AutoInterrupt.__qualname__ = "Git.AutoInterrupt" + + +class _CatFileContentStream: + """Object representing a sized read-only stream returning the contents of + an object. + + This behaves like a stream, but counts the data read and simulates an empty stream + once our sized content region is empty. + + If not all data are read to the end of the object's lifetime, we read the rest to + ensure the underlying stream continues to work. + """ + + __slots__ = ("_stream", "_nbr", "_size") + + def __init__(self, size: int, stream: IO[bytes]) -> None: + self._stream = stream + self._size = size + self._nbr = 0 # Number of bytes read. + + # Special case: If the object is empty, has null bytes, get the final + # newline right away. + if size == 0: + stream.read(1) + # END handle empty streams + + def read(self, size: int = -1) -> bytes: + bytes_left = self._size - self._nbr + if bytes_left == 0: + return b"" + if size > -1: + # Ensure we don't try to read past our limit. + size = min(bytes_left, size) + else: + # They try to read all, make sure it's not more than what remains. + size = bytes_left + # END check early depletion + data = self._stream.read(size) + self._nbr += len(data) + + # Check for depletion, read our final byte to make the stream usable by + # others. + if self._size - self._nbr == 0: + self._stream.read(1) # final newline + # END finish reading + return data + + def readline(self, size: int = -1) -> bytes: + if self._nbr == self._size: + return b"" + + # Clamp size to lowest allowed value. + bytes_left = self._size - self._nbr + if size > -1: + size = min(bytes_left, size) + else: + size = bytes_left + # END handle size + + data = self._stream.readline(size) + self._nbr += len(data) + + # Handle final byte. + if self._size - self._nbr == 0: + self._stream.read(1) + # END finish reading + + return data + + def readlines(self, size: int = -1) -> List[bytes]: + if self._nbr == self._size: + return [] + + # Leave all additional logic to our readline method, we just check the size. + out = [] + nbr = 0 + while True: + line = self.readline() + if not line: + break + out.append(line) + if size > -1: + nbr += len(line) + if nbr > size: + break + # END handle size constraint + # END readline loop + return out + + # skipcq: PYL-E0301 + def __iter__(self) -> "Git.CatFileContentStream": + return self + + def __next__(self) -> bytes: + line = self.readline() + if not line: + raise StopIteration + + return line + + next = __next__ + + def __del__(self) -> None: + bytes_left = self._size - self._nbr + if bytes_left: + # Read and discard - seeking is impossible within a stream. + # This includes any terminating newline. + self._stream.read(bytes_left + 1) + # END handle incomplete read + + +_CatFileContentStream.__name__ = "CatFileContentStream" +_CatFileContentStream.__qualname__ = "Git.CatFileContentStream" + + _USE_SHELL_DEFAULT_MESSAGE = ( "Git.USE_SHELL is deprecated, because only its default value of False is safe. " "It will be removed in a future release." @@ -728,221 +952,9 @@ def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it." ) - class AutoInterrupt: - """Process wrapper that terminates the wrapped process on finalization. - - This kills/interrupts the stored process instance once this instance goes out of - scope. It is used to prevent processes piling up in case iterators stop reading. - - All attributes are wired through to the contained process object. - - The wait method is overridden to perform automatic status code checking and - possibly raise. - """ - - __slots__ = ("proc", "args", "status") - - # If this is non-zero it will override any status code during _terminate, used - # to prevent race conditions in testing. - _status_code_if_terminate: int = 0 - - def __init__(self, proc: Union[None, subprocess.Popen], args: Any) -> None: - self.proc = proc - self.args = args - self.status: Union[int, None] = None - - def _terminate(self) -> None: - """Terminate the underlying process.""" - if self.proc is None: - return - - proc = self.proc - self.proc = None - if proc.stdin: - proc.stdin.close() - if proc.stdout: - proc.stdout.close() - if proc.stderr: - proc.stderr.close() - # Did the process finish already so we have a return code? - try: - if proc.poll() is not None: - self.status = self._status_code_if_terminate or proc.poll() - return - except OSError as ex: - _logger.info("Ignored error after process had died: %r", ex) - - # It can be that nothing really exists anymore... - if os is None or getattr(os, "kill", None) is None: - return - - # Try to kill it. - try: - proc.terminate() - status = proc.wait() # Ensure the process goes away. - - self.status = self._status_code_if_terminate or status - except OSError as ex: - _logger.info("Ignored error after process had died: %r", ex) - # END exception handling - - def __del__(self) -> None: - self._terminate() - - def __getattr__(self, attr: str) -> Any: - return getattr(self.proc, attr) - - # TODO: Bad choice to mimic `proc.wait()` but with different args. - def wait(self, stderr: Union[None, str, bytes] = b"") -> int: - """Wait for the process and return its status code. - - :param stderr: - Previously read value of stderr, in case stderr is already closed. - - :warn: - May deadlock if output or error pipes are used and not handled - separately. - - :raise git.exc.GitCommandError: - If the return status is not 0. - """ - if stderr is None: - stderr_b = b"" - stderr_b = force_bytes(data=stderr, encoding="utf-8") - status: Union[int, None] - if self.proc is not None: - status = self.proc.wait() - p_stderr = self.proc.stderr - else: # Assume the underlying proc was killed earlier or never existed. - status = self.status - p_stderr = None - - def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> bytes: - if stream: - try: - return stderr_b + force_bytes(stream.read()) - except (OSError, ValueError): - return stderr_b or b"" - else: - return stderr_b or b"" - - # END status handling - - if status != 0: - errstr = read_all_from_possibly_closed_stream(p_stderr) - _logger.debug("AutoInterrupt wait stderr: %r" % (errstr,)) - raise GitCommandError(remove_password_if_present(self.args), status, errstr) - return status - - # END auto interrupt - - class CatFileContentStream: - """Object representing a sized read-only stream returning the contents of - an object. - - This behaves like a stream, but counts the data read and simulates an empty - stream once our sized content region is empty. - - If not all data are read to the end of the object's lifetime, we read the - rest to ensure the underlying stream continues to work. - """ - - __slots__ = ("_stream", "_nbr", "_size") - - def __init__(self, size: int, stream: IO[bytes]) -> None: - self._stream = stream - self._size = size - self._nbr = 0 # Number of bytes read. - - # Special case: If the object is empty, has null bytes, get the final - # newline right away. - if size == 0: - stream.read(1) - # END handle empty streams - - def read(self, size: int = -1) -> bytes: - bytes_left = self._size - self._nbr - if bytes_left == 0: - return b"" - if size > -1: - # Ensure we don't try to read past our limit. - size = min(bytes_left, size) - else: - # They try to read all, make sure it's not more than what remains. - size = bytes_left - # END check early depletion - data = self._stream.read(size) - self._nbr += len(data) - - # Check for depletion, read our final byte to make the stream usable by - # others. - if self._size - self._nbr == 0: - self._stream.read(1) # final newline - # END finish reading - return data - - def readline(self, size: int = -1) -> bytes: - if self._nbr == self._size: - return b"" - - # Clamp size to lowest allowed value. - bytes_left = self._size - self._nbr - if size > -1: - size = min(bytes_left, size) - else: - size = bytes_left - # END handle size - - data = self._stream.readline(size) - self._nbr += len(data) - - # Handle final byte. - if self._size - self._nbr == 0: - self._stream.read(1) - # END finish reading - - return data - - def readlines(self, size: int = -1) -> List[bytes]: - if self._nbr == self._size: - return [] - - # Leave all additional logic to our readline method, we just check the size. - out = [] - nbr = 0 - while True: - line = self.readline() - if not line: - break - out.append(line) - if size > -1: - nbr += len(line) - if nbr > size: - break - # END handle size constraint - # END readline loop - return out - - # skipcq: PYL-E0301 - def __iter__(self) -> "Git.CatFileContentStream": - return self - - def __next__(self) -> bytes: - line = self.readline() - if not line: - raise StopIteration - - return line - - next = __next__ + AutoInterrupt = _AutoInterrupt - def __del__(self) -> None: - bytes_left = self._size - self._nbr - if bytes_left: - # Read and discard - seeking is impossible within a stream. - # This includes any terminating newline. - self._stream.read(bytes_left + 1) - # END handle incomplete read + CatFileContentStream = _CatFileContentStream def __init__(self, working_dir: Union[None, PathLike] = None) -> None: """Initialize this instance with: From c6d16d01c54d45be20de40dedae4e9471b96c8ab Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 11:35:08 -0400 Subject: [PATCH 1166/1392] Start on fixing AutoInterrupt/CatFileContentStream aliases This uses `TypeAlias` from the `typing` module, to make it so the assignment statments introduced in #2037 (to set `Git.AutoInterrupt` and `Git.CatFileContentStream` to nonpublic module-level implementations `_AutoInterrupt` and `_CatFileContentStream`) are treated by `mypy` as type aliases rather than as class variables. For details on the problem this partially fixes, see #2038 and: https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases The fix won't work in this form, however, because it attempts to import `TypeAlias` unconditionally from the standard-library `typing` module, which only gained it in Python 3.10. --- git/cmd.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index a6195880d..7015d376f 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -55,6 +55,7 @@ TYPE_CHECKING, TextIO, Tuple, + TypeAlias, Union, cast, overload, @@ -952,9 +953,9 @@ def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it." ) - AutoInterrupt = _AutoInterrupt + AutoInterrupt: TypeAlias = _AutoInterrupt - CatFileContentStream = _CatFileContentStream + CatFileContentStream: TypeAlias = _CatFileContentStream def __init__(self, working_dir: Union[None, PathLike] = None) -> None: """Initialize this instance with: From c6c081230f4b96ca9efce4c9e2478396eaf348c9 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 11:52:36 -0400 Subject: [PATCH 1167/1392] Import `TypeAlias` from `typing_extensions` where needed The standard library `typing` module introduced `TypeAlias` in Python 3.10. This uses it from `typing_extensions` where neederd, by making three changes: - Change the version lower bound for `typing-extensions` from 3.7.4.3 to 3.10.0.2, since 3.7.4.3 doesn't offer `TypeAlias`. (The reason not to go higher, to major version 4, is that it no longer supports versions of Python lower than 3.9, but we currently support Python 3.7 and Python 3.8.) - Require the `typing-extensions` dependency when using Python versions lower than 3.10, rather than only lower than 3.7 as before. - Conditionally import `TypeAlias` (in the `git.cmd` module) from either `typing` or `type_extensions` depending on the Python version, using a pattern that `mypy` and other type checkers recognize statically. Together with the preceding commit, this fixes #2038. (This is approach (2) described there.) --- git/cmd.py | 6 +++++- requirements.txt | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 7015d376f..6deded04b 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -55,12 +55,16 @@ TYPE_CHECKING, TextIO, Tuple, - TypeAlias, Union, cast, overload, ) +if sys.version_info >= (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias + from git.types import Literal, PathLike, TBD if TYPE_CHECKING: diff --git a/requirements.txt b/requirements.txt index 7159416a9..61d8403b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ gitdb>=4.0.1,<5 -typing-extensions>=3.7.4.3;python_version<"3.8" +typing-extensions>=3.10.0.2;python_version<"3.10" From 8905e9e8f0244414fbff90c19b144d02b74af3b3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 12:33:56 -0400 Subject: [PATCH 1168/1392] Fix CI `mypy` command on free-threaded Python When the version is represented as `3.13t`, the `--python-version` option needs an operand of `3.13`, not `3.13t`. (This, and the fix here, will automatically apply to later threaded Pythons, such as 3.14t, too.) --- .github/workflows/pythonpackage.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 9fd660c6b..c4fbef48e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -97,10 +97,11 @@ jobs: - name: Check types with mypy run: | - mypy --python-version=${{ matrix.python-version }} + mypy --python-version="${PYTHON_VERSION%t}" env: MYPY_FORCE_COLOR: "1" TERM: "xterm-256color" # For color: https://github.com/python/mypy/issues/13817 + PYTHON_VERSION: ${{ matrix.python-version }} # With new versions of mypy new issues might arise. This is a problem if there is # nobody able to fix them, so we have to ignore errors until that changes. continue-on-error: true From b45ae86aaaab25ccd07ee183382a0e78a92f65a2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 13:10:47 -0400 Subject: [PATCH 1169/1392] Add an explanatory comment for omitting "t" Since the `${var%pattern}` syntax may not be immediately obvious. --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index c4fbef48e..e7cb06cc0 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -97,7 +97,7 @@ jobs: - name: Check types with mypy run: | - mypy --python-version="${PYTHON_VERSION%t}" + mypy --python-version="${PYTHON_VERSION%t}" # Version only, with no "t" for free-threaded. env: MYPY_FORCE_COLOR: "1" TERM: "xterm-256color" # For color: https://github.com/python/mypy/issues/13817 From ba58f40d385e5f371c468470752a4d5aa0b16789 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 13:37:22 -0400 Subject: [PATCH 1170/1392] Split Cygwin CI into two test jobs One job is for all tests except the `performance` tests, while the other job is for only the `performance` tests. The idea is to decrease the total time it takes for all CI jobs to complete in most cases, by splitting the long-running (currently usually about 13 minute) Cygwin job into two less-long jobs. --- .github/workflows/cygwin-test.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 572a9197e..d73f0522e 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -10,6 +10,14 @@ jobs: runs-on: windows-latest strategy: + matrix: + selection: [fast, perf] + include: + - selection: fast + additional-pytest-args: --ignore=test/performance + - selection: perf + additional-pytest-args: test/performance/ + fail-fast: false env: @@ -87,4 +95,4 @@ jobs: - name: Test with pytest run: | - pytest --color=yes -p no:sugar --instafail -vv + pytest --color=yes -p no:sugar --instafail -vv ${{ matrix.additional-pytest-args }} From 414de64b095282ce39b9dd34876167e607b331cf Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 14:13:25 -0400 Subject: [PATCH 1171/1392] Show differing `pytest` arguments in step name In the "Test with pytest" step of the Cygwin test jobs. This is to distinguish the newly split jobs from each other more clearly when glancing at their steps for a run. --- .github/workflows/cygwin-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index d73f0522e..3c1eb3dc0 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -93,6 +93,6 @@ jobs: python --version python -c 'import os, sys; print(f"sys.platform={sys.platform!r}, os.name={os.name!r}")' - - name: Test with pytest + - name: Test with pytest (${{ matrix.additional-pytest-args }}) run: | pytest --color=yes -p no:sugar --instafail -vv ${{ matrix.additional-pytest-args }} From a6c623eecf12eb525ad115b0be75c7449c4f8efe Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 14:18:14 -0400 Subject: [PATCH 1172/1392] Small stylistic consistency tweak This removes an unnecessary trailing slash that I had not used consistently anyway. --- .github/workflows/cygwin-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 3c1eb3dc0..7c3eeedca 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -16,7 +16,7 @@ jobs: - selection: fast additional-pytest-args: --ignore=test/performance - selection: perf - additional-pytest-args: test/performance/ + additional-pytest-args: test/performance fail-fast: false From 31e1c035e1af514d5d2a9ed6630f71663df7b265 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 15:01:21 -0400 Subject: [PATCH 1173/1392] Use static notation for `setuptools` in installation test The `VirtualEnvironment` test helper is used in multiple places, but it is only told to install/upgrade `pip` when used from `test_installation`. It implements the functionality it would ideally obtain by `upgrade_deps`, since the `upgrade_deps` parameter is only avaiilable in `venv` when using Python 3.9 and later. When `pip` is installed, `upgrade_deps` would install `setuptools` when using Python 3.11 or lower, but not when using Python 3.12 or higher. `VirtualEnvironment` does the same. (The reason for this is not just to avoid needlessly departing from what `upgrade_deps` would do. Rather, it should not generally be necessary to have `setuptools` installed for package management since Python 3.12, and if it were necessary then this would a bug we would want to detect while running tests.) Previously this conditional specification of `setuptools` was done by building different lists of package arguments to pass to `pip`, by checking `sys.version_info` to decide whether to append the string `setuptools`. This commit changes how it is done, to use a static list of package arguments instead. (The Python intepreter path continues to be obtained dynamically, but all its positional arguments, including those specifying packages, are now string literals.) The conditional `setuptools` requirement is now expressed statically using notation recognized by `pip`, as the string `setuptools; python_version<"3.12"`. --- test/lib/helper.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index 5d91447ea..241d27341 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -415,9 +415,15 @@ def __init__(self, env_dir, *, with_pip): if with_pip: # The upgrade_deps parameter to venv.create is 3.9+ only, so do it this way. - command = [self.python, "-m", "pip", "install", "--upgrade", "pip"] - if sys.version_info < (3, 12): - command.append("setuptools") + command = [ + self.python, + "-m", + "pip", + "install", + "--upgrade", + "pip", + 'setuptools; python_version<"3.12"', + ] subprocess.check_output(command) @property From 727f4e9e54362d0356066a26d0c22028a465cccd Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 15:21:00 -0400 Subject: [PATCH 1174/1392] Use static notation for `setuptools` in CI `pip` commands `setuptools` may potentially be needed for installation to work fully as desired prior to Python 3.12, so in those versions it is installed automatically in any virtual environment that is created with `pip` available. This is a behavior of the `venv` module that is not specific to CI. However, on CI we upgrade packages that are preinstalled in the virtual environment, or that we may otherwise wish to be present. This took the form of unconditionally installing/upgrading the `pip` and `wheel` packages, but conditionally upgrading the `setuptools` package only if we find that it is already installed. This commit changes the behavior to statically specify the same list of package specifications to `pip` in all environments and in all versions of Python, but to use the static notation recognized by `pip` to indicate that `setuptools` is to be instaled/upgraded only if the Python version is strictly less than Python 3.12. This seems to be more readable. It also avoids using unquoted shell parameter expansion in a way that is potentially confusing (for example, if we were running our CI script steps through ShellCheck, then it would automatically balk at that construction). It is also more consistent with how `test_installation` sets up its environment (especially since 31e1c03, but actually even before that, because it was still conditioning `setuptools` on the Python version rather than whether it was already installed). Finally, this behavior is what the preexisting comment already described. This also adjusts the shell quoting style slightly in other related commands (in the same workflows) that pass package specifications to `pip`, for consistency. (For now, `".[test]"` rather than `.[test]` remains written in the readme because it works in `cmd.exe` as well as other shells, but it may be changed there in the future too.) --- .github/workflows/alpine-test.yml | 4 ++-- .github/workflows/cygwin-test.yml | 4 ++-- .github/workflows/pythonpackage.yml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 513c65bb8..a3361798d 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -55,12 +55,12 @@ jobs: run: | # Get the latest pip, wheel, and prior to Python 3.12, setuptools. . .venv/bin/activate - python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel + python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel - name: Install project and test dependencies run: | . .venv/bin/activate - pip install ".[test]" + pip install '.[test]' - name: Show version and platform information run: | diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 7c3eeedca..6943db09c 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -79,11 +79,11 @@ jobs: - name: Update PyPA packages run: | # Get the latest pip, wheel, and prior to Python 3.12, setuptools. - python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel + python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel - name: Install project and test dependencies run: | - pip install ".[test]" + pip install '.[test]' - name: Show version and platform information run: | diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index e7cb06cc0..c56d45df7 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -72,11 +72,11 @@ jobs: - name: Update PyPA packages run: | # Get the latest pip, wheel, and prior to Python 3.12, setuptools. - python -m pip install -U pip $(pip freeze --all | grep -ow ^setuptools) wheel + python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel - name: Install project and test dependencies run: | - pip install ".[test]" + pip install '.[test]' - name: Show version and platform information run: | @@ -114,5 +114,5 @@ jobs: - name: Documentation if: matrix.python-version != '3.7' run: | - pip install ".[doc]" + pip install '.[doc]' make -C doc html From 2b85cd1d7ccbbc596cf0d3149bdc854498ebe35e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 16:35:08 -0400 Subject: [PATCH 1175/1392] Fix ambiguous `_safer_popen_windows` comment This fixes some ambiguous wording in a comment in `_safer_popen_wording`, where it was unclear if the secondary problem -- where it would be possible to run a wrong `cmd.exe`-type shell -- would happen under two separate circumstances, or only when both circumstances occurred together. This adjusts its wording to make clear that it is the latter. This also fixes a minor typo in another `_safer_popen_windows` comment. This might be viewed as building on the improvements in b9d9e56 (#1859), but the changes here are to comments only. --- git/cmd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 6deded04b..71096197c 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -273,12 +273,12 @@ def _safer_popen_windows( if shell: # The original may be immutable, or the caller may reuse it. Mutate a copy. env = {} if env is None else dict(env) - env["NoDefaultCurrentDirectoryInExePath"] = "1" # The "1" can be an value. + env["NoDefaultCurrentDirectoryInExePath"] = "1" # The "1" can be any value. # When not using a shell, the current process does the search in a # CreateProcessW API call, so the variable must be set in our environment. With # a shell, that's unnecessary if https://github.com/python/cpython/issues/101283 - # is patched. In Python versions where it is unpatched, and in the rare case the + # is patched. In Python versions where it is unpatched, in the rare case the # ComSpec environment variable is unset, the search for the shell itself is # unsafe. Setting NoDefaultCurrentDirectoryInExePath in all cases, as done here, # is simpler and protects against that. (As above, the "1" can be any value.) From 253099fe91744801c39b41527e126c85c17ec003 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 18:28:17 -0400 Subject: [PATCH 1176/1392] Clarify `USE_SHELL` warning helper signature This is a minor refactor of how `_warn_use_shell` can be, and is, invoked. The `_warn_use_shell` helper function in `git.cmd` takes a single `bool`-valued argument `extra_danger`, which is conceptually associated with having a `True` value of `USE_SHELL`, but the association is not necessarily obvious. Specifically: - For the warning given when reading `USE_SHELL` on the `Git` class or through an instance, `extra_danger` is always `False`. This is so even if the `USE_SHELL` value is currently `True`, because the danger that arises from `True` occurs internally. - For the warning given when writing `USE_SHELL`, which can only be done on the `Git` class and not on or through an instance, `extra_danger` is the value set for the attribute. This is because setting `USE_SHELL` to `True` incurs the danger described in #1896. When reading the code, which passed `extra_danger` positionally, the meaning of the parameter may not always have been obvious. This makes the `extra_danger` parameter keyword-only, and passes it by keyword in all invocations, so that its meaning is clearer. --- git/cmd.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 71096197c..15d7820df 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -550,7 +550,7 @@ def __del__(self) -> None: ) -def _warn_use_shell(extra_danger: bool) -> None: +def _warn_use_shell(*, extra_danger: bool) -> None: warnings.warn( _USE_SHELL_DANGER_MESSAGE if extra_danger else _USE_SHELL_DEFAULT_MESSAGE, DeprecationWarning, @@ -566,12 +566,12 @@ class _GitMeta(type): def __getattribute(cls, name: str) -> Any: if name == "USE_SHELL": - _warn_use_shell(False) + _warn_use_shell(extra_danger=False) return super().__getattribute__(name) def __setattr(cls, name: str, value: Any) -> Any: if name == "USE_SHELL": - _warn_use_shell(value) + _warn_use_shell(extra_danger=value) super().__setattr__(name, value) if not TYPE_CHECKING: @@ -988,7 +988,7 @@ def __init__(self, working_dir: Union[None, PathLike] = None) -> None: def __getattribute__(self, name: str) -> Any: if name == "USE_SHELL": - _warn_use_shell(False) + _warn_use_shell(extra_danger=False) return super().__getattribute__(name) def __getattr__(self, name: str) -> Any: From e5a2db58d4d7fee765bd1146647bf2c09b026ce8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 19:24:31 -0400 Subject: [PATCH 1177/1392] Test `ConfigParser` with whitespace outside the value In both the: - Unquoted case, where extra whitespace is at the edges of what can be parsed as the value. - Quoted case, where extra whitespace is next to but outside of the quotes. The case where the whitespace is at the edges of the quoted value and *inside* the quotes, and thus part of the value, is already covered in #2036. (That is merely renamed here, to distinguish it.) --- test/fixtures/git_config_with_extra_whitespace | 2 ++ ...espace => git_config_with_quotes_whitespace_inside} | 0 .../fixtures/git_config_with_quotes_whitespace_outside | 2 ++ test/test_config.py | 10 +++++++++- 4 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/git_config_with_extra_whitespace rename test/fixtures/{git_config_with_quotes_whitespace => git_config_with_quotes_whitespace_inside} (100%) create mode 100644 test/fixtures/git_config_with_quotes_whitespace_outside diff --git a/test/fixtures/git_config_with_extra_whitespace b/test/fixtures/git_config_with_extra_whitespace new file mode 100644 index 000000000..0f727cb5d --- /dev/null +++ b/test/fixtures/git_config_with_extra_whitespace @@ -0,0 +1,2 @@ +[init] + defaultBranch = trunk diff --git a/test/fixtures/git_config_with_quotes_whitespace b/test/fixtures/git_config_with_quotes_whitespace_inside similarity index 100% rename from test/fixtures/git_config_with_quotes_whitespace rename to test/fixtures/git_config_with_quotes_whitespace_inside diff --git a/test/fixtures/git_config_with_quotes_whitespace_outside b/test/fixtures/git_config_with_quotes_whitespace_outside new file mode 100644 index 000000000..4b1615a51 --- /dev/null +++ b/test/fixtures/git_config_with_quotes_whitespace_outside @@ -0,0 +1,2 @@ +[init] + defaultBranch = "trunk" diff --git a/test/test_config.py b/test/test_config.py index 671f34046..879b98365 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -398,6 +398,10 @@ def test_complex_aliases(self): self._to_memcache(fixture_path(".gitconfig")).getvalue(), ) + def test_config_with_extra_whitespace(self): + cr = GitConfigParser(fixture_path("git_config_with_extra_whitespace"), read_only=True) + self.assertEqual(cr.get("init", "defaultBranch"), "trunk") + def test_empty_config_value(self): cr = GitConfigParser(fixture_path("git_config_with_empty_value"), read_only=True) @@ -413,9 +417,13 @@ def test_config_with_quotes(self): self.assertEqual(cr.get("user", "email"), "cveal05@gmail.com") def test_config_with_quotes_with_literal_whitespace(self): - cr = GitConfigParser(fixture_path("git_config_with_quotes_whitespace"), read_only=True) + cr = GitConfigParser(fixture_path("git_config_with_quotes_whitespace_inside"), read_only=True) self.assertEqual(cr.get("core", "commentString"), "# ") + def test_config_with_quotes_with_whitespace_outside_value(self): + cr = GitConfigParser(fixture_path("git_config_with_quotes_whitespace_outside"), read_only=True) + self.assertEqual(cr.get("init", "defaultBranch"), "trunk") + def test_get_values_works_without_requiring_any_other_calls_first(self): file_obj = self._to_memcache(fixture_path("git_config_multiple")) cr = GitConfigParser(file_obj, read_only=True) From 4ebe407493363a814f590a97f3734364e6c5c1e1 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 22:33:35 -0400 Subject: [PATCH 1178/1392] Test that ConfigParser treats empty quotes as an empty value The ConfigParser has supported this for a long time, but it is now done redundantly since #2035. This adds a test for it, both to make clearer that it is intended to work and to allow verifying that it continues to hold once the legacy special-casing for it is removed. --- test/fixtures/git_config_with_empty_quotes | 2 ++ test/test_config.py | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 test/fixtures/git_config_with_empty_quotes diff --git a/test/fixtures/git_config_with_empty_quotes b/test/fixtures/git_config_with_empty_quotes new file mode 100644 index 000000000..f11fe4248 --- /dev/null +++ b/test/fixtures/git_config_with_empty_quotes @@ -0,0 +1,2 @@ +[core] + filemode = "" diff --git a/test/test_config.py b/test/test_config.py index 879b98365..76b918a54 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -416,6 +416,10 @@ def test_config_with_quotes(self): self.assertEqual(cr.get("user", "name"), "Cody Veal") self.assertEqual(cr.get("user", "email"), "cveal05@gmail.com") + def test_config_with_empty_quotes(self): + cr = GitConfigParser(fixture_path("git_config_with_empty_quotes"), read_only=True) + self.assertEqual(cr.get("core", "filemode"), "", "quotes can form a literal empty string as value") + def test_config_with_quotes_with_literal_whitespace(self): cr = GitConfigParser(fixture_path("git_config_with_quotes_whitespace_inside"), read_only=True) self.assertEqual(cr.get("core", "commentString"), "# ") From 2f225244286567b72c865fc7c0219c7dc043575f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 22:59:24 -0400 Subject: [PATCH 1179/1392] Remove explicit empty `""` handling in ConfigParser Because literal `""` is a special case of `"..."` as parsed since #2035. --- git/config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/git/config.py b/git/config.py index 1d58db53a..cdcc32e7b 100644 --- a/git/config.py +++ b/git/config.py @@ -501,9 +501,6 @@ def string_decode(v: str) -> str: if pos != -1 and optval[pos - 1].isspace(): optval = optval[:pos] optval = optval.strip() - if optval == '""': - optval = "" - # END handle empty string optname = self.optionxform(optname.rstrip()) if len(optval) > 1 and optval[0] == '"' and optval[-1] != '"': is_multi_line = True From c8e4aa0f30d06ea2437539a5d56eede1ffa11432 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 23:15:05 -0400 Subject: [PATCH 1180/1392] Refactor quote parsing, to prepare for more checks This refactors ConfigParser double-quote parsing near the single line double-quoted value parsing code, so that: - Code that parses the name is less intermixed with code that parses the value. - Conditional logic is less duplicated. - The `END` comment notation appears next to the code it describes. - The final `else` can be turned into one or more `elif` followed by `else` to cover different cases of `"..."` differently. (But those are not added here. This commit is purely a refactoring.) (The `pass` suite when `len(optval) < 2 or optval[0] != '"'` is awkward and not really justified right now, but it looks like it may be able to help with readabilty and help keep nesting down when new `elif` cases are added.) --- git/config.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/git/config.py b/git/config.py index cdcc32e7b..afec498f2 100644 --- a/git/config.py +++ b/git/config.py @@ -496,18 +496,23 @@ def string_decode(v: str) -> str: if mo: # We might just have handled the last line, which could contain a quotation we want to remove. optname, vi, optval = mo.group("option", "vi", "value") + optname = self.optionxform(optname.rstrip()) + if vi in ("=", ":") and ";" in optval and not optval.strip().startswith('"'): pos = optval.find(";") if pos != -1 and optval[pos - 1].isspace(): optval = optval[:pos] optval = optval.strip() - optname = self.optionxform(optname.rstrip()) - if len(optval) > 1 and optval[0] == '"' and optval[-1] != '"': + + if len(optval) < 2 or optval[0] != '"': + pass # Nothing to treat as opening quotation. + elif optval[-1] != '"': is_multi_line = True optval = string_decode(optval[1:]) - elif len(optval) > 1 and optval[0] == '"' and optval[-1] == '"': - optval = optval[1:-1] # END handle multi-line + else: + optval = optval[1:-1] + # Preserves multiple values for duplicate optnames. cursect.add(optname, optval) else: From 7bcea08873ca4052ab743f9e2f7cff42e7fe62d8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 7 Jun 2025 23:59:46 -0400 Subject: [PATCH 1181/1392] Add tests of ConfigParser with `"..."` with `"` or `\` inside These are cases where just removing the outer quotes without doing anything to the text inside does not give the correct result, and where keeping the quotes may be preferable, in that it was the long-standing behavior of `GitConfigParser`. That this was the long-standing behavior may justify bringing it back when the `"`-`"`-enclosed text contains such characters, but it does not justify preserving it indefinitely: it will still be better to parse the escape sequences, at least in the type case that all of them in a value's representation are well-formed. --- test/fixtures/git_config_with_quotes_escapes | 9 +++++++++ test/test_config.py | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 test/fixtures/git_config_with_quotes_escapes diff --git a/test/fixtures/git_config_with_quotes_escapes b/test/fixtures/git_config_with_quotes_escapes new file mode 100644 index 000000000..33332c221 --- /dev/null +++ b/test/fixtures/git_config_with_quotes_escapes @@ -0,0 +1,9 @@ +[custom] + hasnewline = "first\nsecond" + hasbackslash = "foo\\bar" + hasquote = "ab\"cd" + hastrailingbackslash = "word\\" + hasunrecognized = "p\qrs" + hasunescapedquotes = "ab"cd"e" + ordinary = "hello world" + unquoted = good evening diff --git a/test/test_config.py b/test/test_config.py index 76b918a54..8e1007d9e 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -428,6 +428,26 @@ def test_config_with_quotes_with_whitespace_outside_value(self): cr = GitConfigParser(fixture_path("git_config_with_quotes_whitespace_outside"), read_only=True) self.assertEqual(cr.get("init", "defaultBranch"), "trunk") + def test_config_with_quotes_containing_escapes(self): + """For now just suppress quote removal. But it would be good to interpret most of these.""" + cr = GitConfigParser(fixture_path("git_config_with_quotes_escapes"), read_only=True) + + # These can eventually be supported by substituting the represented character. + self.assertEqual(cr.get("custom", "hasnewline"), R'"first\nsecond"') + self.assertEqual(cr.get("custom", "hasbackslash"), R'"foo\\bar"') + self.assertEqual(cr.get("custom", "hasquote"), R'"ab\"cd"') + self.assertEqual(cr.get("custom", "hastrailingbackslash"), R'"word\\"') + self.assertEqual(cr.get("custom", "hasunrecognized"), R'"p\qrs"') + + # It is less obvious whether and what to eventually do with this. + self.assertEqual(cr.get("custom", "hasunescapedquotes"), '"ab"cd"e"') + + # Cases where quote removal is clearly safe should happen even after those. + self.assertEqual(cr.get("custom", "ordinary"), "hello world") + + # Cases without quotes should still parse correctly even after those, too. + self.assertEqual(cr.get("custom", "unquoted"), "good evening") + def test_get_values_works_without_requiring_any_other_calls_first(self): file_obj = self._to_memcache(fixture_path("git_config_multiple")) cr = GitConfigParser(file_obj, read_only=True) From f2b80410e96a256aed044fba0387eab0440a1525 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 8 Jun 2025 00:09:49 -0400 Subject: [PATCH 1182/1392] Don't remove quotes if `\` or `"` are present inside This is for single line quoting in the ConfigParser. This leaves the changes in #2035 (as adjusted in #2036) intact for the cases where it addressed #1923: when the `...` in `"..."` (appearing in the value position on a single `{name} = {value}"` line) has no occurrences of `\` or `"`, quote removal is enough. But when `\` or `"` does appear, this suppresses quote removal. This is with the idea that, while it would be better to interpret such lines as Git does, we do not yet do that, so it is preferable to return the same results we have in the past (which some programs may already be handling themselves). This should make the test introduced in the preceding commit pass. But it will be even better to support more syntax, at least well-formed escapes. As noted in the test, both the test and the code under test can be adjusted for that. (See comments in #2035 for context.) --- git/config.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/git/config.py b/git/config.py index afec498f2..5fc099a27 100644 --- a/git/config.py +++ b/git/config.py @@ -505,13 +505,16 @@ def string_decode(v: str) -> str: optval = optval.strip() if len(optval) < 2 or optval[0] != '"': - pass # Nothing to treat as opening quotation. + # Does not open quoting. + pass elif optval[-1] != '"': + # Opens quoting and does not close: appears to start multi-line quoting. is_multi_line = True optval = string_decode(optval[1:]) - # END handle multi-line - else: + elif optval.find("\\", 1, -1) == -1 and optval.find('"', 1, -1) == -1: + # Opens and closes quoting. Single line, and all we need is quote removal. optval = optval[1:-1] + # TODO: Handle other quoted content, especially well-formed backslash escapes. # Preserves multiple values for duplicate optnames. cursect.add(optname, optval) From 1b79d449c90cc6fca8220246e2de159d24666837 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 8 Jun 2025 14:33:58 -0400 Subject: [PATCH 1183/1392] Improve line-ending consistency in requirements files `requirements-dev.txt`, but none of the others, was tracked with Windows-style (CRLF) line endings. This appears to have been the case since it was introduced in a1b7634 (as `dev-requirements.txt`) and not to be intentional. This only changes how it is stored in the repository. This does not change `.gitattributes` (it is not forced to have LF line endings if automatic line-ending conversions are configured in Git). --- requirements-dev.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f626644af..01cb2d040 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,8 @@ --r requirements.txt --r test-requirements.txt - -# For additional local testing/linting - to be added elsewhere eventually. -ruff -shellcheck -pytest-icdiff -# pytest-profiling +-r requirements.txt +-r test-requirements.txt + +# For additional local testing/linting - to be added elsewhere eventually. +ruff +shellcheck +pytest-icdiff +# pytest-profiling From 6f4f7f5137d63facb61eae2955e6c0801c71b7b5 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 8 Jun 2025 15:05:03 -0400 Subject: [PATCH 1184/1392] Update Ruff configuration This resolves two warnings about Ruff configuration, by: - No longer setting `ignore-init-module-imports = true` explicitly, which was deprecated since `ruff` 0.4.4. We primarily use `ruff` via `pre-commit`, for which this deprecation has applied since we upgraded the version in `.pre-commit-config.yaml` from 0.4.3 to 0.6.0 in d1582d1 (#1953). We continue to list `F401` ("Module imported but unused") as not automatically fixable, to avoid inadvertently removing imports that may be needed. See also: https://docs.astral.sh/ruff/settings/#lint_ignore-init-module-imports - Rename the rule `TCH004` to `TC004`, since `TCH004` is the old name that may eventually be removed and that is deprecated since 0.8.0. We upgraded `ruff` in `.pre-commit-config.yml` again in b7ce712 (#2031), from 0.6.0 to 0.11.12, at which point this deprecation applied. See also https://astral.sh/blog/ruff-v0.8.0. These changes make those configuration-related warnings go away, and no new diagnostics (errors/warnings) are produced when running `ruff check` or `pre-commit run --all-files`. No F401-related diagnostics are triggered when testing with explicit `ignore-init-module-imports = false`, in preview mode or otherwise. In addition, this commit makes two changes that are not needed to resolve warnings: - Stop excluding `E203` ("Whitespace before ':'"). That diagnostic is no longer failing with the current code here in the current version of `ruff`, and code changes that would cause it to fail would likely be accidentally mis-st - Add the version lower bound `>=0.8` for `ruff` in `requirements-dev.txt`. That file is rarely used, as noted in a8a73ff7 (#1871), but as long as we have it, there may be a benefit to excluding dependency versions for which our configuration is no longer compatible. This is the only change in this commit outside of `pyproject.toml`. --- pyproject.toml | 10 ++++------ requirements-dev.txt | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 090972eed..0097e9951 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,16 +60,14 @@ lint.select = [ # "UP", # See: https://docs.astral.sh/ruff/rules/#pyupgrade-up ] lint.extend-select = [ - # "A", # See: https://pypi.org/project/flake8-builtins - "B", # See: https://pypi.org/project/flake8-bugbear - "C4", # See: https://pypi.org/project/flake8-comprehensions - "TCH004", # See: https://docs.astral.sh/ruff/rules/runtime-import-in-type-checking-block/ + # "A", # See: https://pypi.org/project/flake8-builtins + "B", # See: https://pypi.org/project/flake8-bugbear + "C4", # See: https://pypi.org/project/flake8-comprehensions + "TC004", # See: https://docs.astral.sh/ruff/rules/runtime-import-in-type-checking-block/ ] lint.ignore = [ - "E203", # Whitespace before ':' "E731", # Do not assign a `lambda` expression, use a `def` ] -lint.ignore-init-module-imports = true lint.unfixable = [ "F401", # Module imported but unused ] diff --git a/requirements-dev.txt b/requirements-dev.txt index 01cb2d040..066b192b8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ -r test-requirements.txt # For additional local testing/linting - to be added elsewhere eventually. -ruff +ruff >=0.8 shellcheck pytest-icdiff # pytest-profiling From a36b8a5a726ee5a17cfd492e49c0b0b2a05b2136 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 8 Jun 2025 16:25:17 -0400 Subject: [PATCH 1185/1392] Always use a `def` instead of assigning a `lambda` This stops listing Ruff rule `E731` ("Do not assign a `lambda` expression, use a `def`") as ignored, and fixes all occurrences of it: - Spacing is manually adjusted so that readability is not harmed, while still satisfying the current formatting conventions. - Although the affected test modules do not currently use type annotations, the non-test modules do. Some of the lambdas already had type annotations, by annotating the variable itself with an expression formed by subscripting `Callable`. This change preserves them, converting them to paramter and return type annotations in the resulting `def`. Where such type annotations were absent (in lambdas in non-test modules), or partly absent, all missing annotations are added to the `def`. - Unused paramters are prefixed with a `_`. - `IndexFile.checkout` assigned a lambda to `make_exc`, whose body was somewhat difficult to read. Separately from converting it to a `def`, this refactors the expression in the `return` statement to use code like `(x, *ys)` in place of `(x,) + tuple(ys)`. This change does not appear to have introduced (nor fixed) any `mypy` errors. This only affects lambdas that were assigned directly to variables. Other lambda expressions remain unchanged. --- git/index/base.py | 19 +++++++++++++++---- git/objects/tree.py | 4 +++- pyproject.toml | 2 +- test/test_fun.py | 6 +++++- test/test_index.py | 5 ++++- test/test_tree.py | 10 ++++++++-- 6 files changed, 36 insertions(+), 10 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index a95762dca..7cc9d3ade 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -530,7 +530,10 @@ def unmerged_blobs(self) -> Dict[PathLike, List[Tuple[StageType, Blob]]]: stage. That is, a file removed on the 'other' branch whose entries are at stage 3 will not have a stage 3 entry. """ - is_unmerged_blob = lambda t: t[0] != 0 + + def is_unmerged_blob(t: Tuple[StageType, Blob]) -> bool: + return t[0] != 0 + path_map: Dict[PathLike, List[Tuple[StageType, Blob]]] = {} for stage, blob in self.iter_blobs(is_unmerged_blob): path_map.setdefault(blob.path, []).append((stage, blob)) @@ -690,12 +693,17 @@ def _store_path(self, filepath: PathLike, fprogress: Callable) -> BaseIndexEntry This must be ensured in the calling code. """ st = os.lstat(filepath) # Handles non-symlinks as well. + if S_ISLNK(st.st_mode): # In PY3, readlink is a string, but we need bytes. # In PY2, it was just OS encoded bytes, we assumed UTF-8. - open_stream: Callable[[], BinaryIO] = lambda: BytesIO(force_bytes(os.readlink(filepath), encoding=defenc)) + def open_stream() -> BinaryIO: + return BytesIO(force_bytes(os.readlink(filepath), encoding=defenc)) else: - open_stream = lambda: open(filepath, "rb") + + def open_stream() -> BinaryIO: + return open(filepath, "rb") + with open_stream() as stream: fprogress(filepath, False, filepath) istream = self.repo.odb.store(IStream(Blob.type, st.st_size, stream)) @@ -1336,8 +1344,11 @@ def handle_stderr(proc: "Popen[bytes]", iter_checked_out_files: Iterable[PathLik kwargs["as_process"] = True kwargs["istream"] = subprocess.PIPE proc = self.repo.git.checkout_index(args, **kwargs) + # FIXME: Reading from GIL! - make_exc = lambda: GitCommandError(("git-checkout-index",) + tuple(args), 128, proc.stderr.read()) + def make_exc() -> GitCommandError: + return GitCommandError(("git-checkout-index", *args), 128, proc.stderr.read()) + checked_out_files: List[PathLike] = [] for path in paths: diff --git a/git/objects/tree.py b/git/objects/tree.py index 09184a781..1845d0d0d 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -50,7 +50,9 @@ # -------------------------------------------------------- -cmp: Callable[[str, str], int] = lambda a, b: (a > b) - (a < b) + +def cmp(a: str, b: str) -> int: + return (a > b) - (a < b) class TreeModifier: diff --git a/pyproject.toml b/pyproject.toml index 0097e9951..58ed81f17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ lint.extend-select = [ "TC004", # See: https://docs.astral.sh/ruff/rules/runtime-import-in-type-checking-block/ ] lint.ignore = [ - "E731", # Do not assign a `lambda` expression, use a `def` + # If it becomes necessary to ignore any rules, list them here. ] lint.unfixable = [ "F401", # Module imported but unused diff --git a/test/test_fun.py b/test/test_fun.py index b8593b400..a456b8aab 100644 --- a/test/test_fun.py +++ b/test/test_fun.py @@ -243,6 +243,7 @@ def test_tree_traversal(self): B_old = self.rorepo.tree("1f66cfbbce58b4b552b041707a12d437cc5f400a") # old base tree # Two very different trees. + entries = traverse_trees_recursive(odb, [B_old.binsha, H.binsha], "") self._assert_tree_entries(entries, 2) @@ -251,7 +252,10 @@ def test_tree_traversal(self): self._assert_tree_entries(oentries, 2) # Single tree. - is_no_tree = lambda i, d: i.type != "tree" + + def is_no_tree(i, _d): + return i.type != "tree" + entries = traverse_trees_recursive(odb, [B.binsha], "") assert len(entries) == len(list(B.traverse(predicate=is_no_tree))) self._assert_tree_entries(entries, 1) diff --git a/test/test_index.py b/test/test_index.py index c42032e70..cf3b90fa6 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -330,7 +330,10 @@ def test_index_file_from_tree(self, rw_repo): assert len([e for e in three_way_index.entries.values() if e.stage != 0]) # ITERATE BLOBS - merge_required = lambda t: t[0] != 0 + + def merge_required(t): + return t[0] != 0 + merge_blobs = list(three_way_index.iter_blobs(merge_required)) assert merge_blobs assert merge_blobs[0][0] in (1, 2, 3) diff --git a/test/test_tree.py b/test/test_tree.py index 73158113d..7ba93bd36 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -126,12 +126,18 @@ def test_traverse(self): assert len(list(root)) == len(list(root.traverse(depth=1))) # Only choose trees. - trees_only = lambda i, d: i.type == "tree" + + def trees_only(i, _d): + return i.type == "tree" + trees = list(root.traverse(predicate=trees_only)) assert len(trees) == len([i for i in root.traverse() if trees_only(i, 0)]) # Test prune. - lib_folder = lambda t, d: t.path == "lib" + + def lib_folder(t, _d): + return t.path == "lib" + pruned_trees = list(root.traverse(predicate=trees_only, prune=lib_folder)) assert len(pruned_trees) < len(trees) From 1d808917168ee292a9e4c5fb575a79133d253bb3 Mon Sep 17 00:00:00 2001 From: David Date: Wed, 11 Jun 2025 14:05:31 +0200 Subject: [PATCH 1186/1392] fix updating submodules with relative urls This fixes running repo.update_submodules(init=True) on repositories that are using relative for the modules. Fixes #730 --- git/objects/submodule/base.py | 5 +++++ test/test_submodule.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index fa60bcdaf..0e55b8fa9 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -353,6 +353,11 @@ def _clone_repo( os.makedirs(module_abspath_dir) module_checkout_path = osp.join(str(repo.working_tree_dir), path) + if url.startswith("../"): + remote_name = repo.active_branch.tracking_branch().remote_name + repo_remote_url = repo.remote(remote_name).url + url = os.path.join(repo_remote_url, url) + clone = git.Repo.clone_from( url, module_checkout_path, diff --git a/test/test_submodule.py b/test/test_submodule.py index d88f9dab0..f44f086c2 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -753,6 +753,22 @@ def test_add_empty_repo(self, rwdir): ) # END for each checkout mode + @with_rw_directory + @_patch_git_config("protocol.file.allow", "always") + def test_update_submodule_with_relative_path(self, rwdir): + repo_path = osp.join(rwdir, "parent") + repo = git.Repo.init(repo_path) + module_repo_path = osp.join(rwdir, "module") + module_repo = git.Repo.init(module_repo_path) + module_repo.git.commit(m="test", allow_empty=True) + repo.git.submodule("add", "../module", "module") + repo.index.commit("add submodule") + + cloned_repo_path = osp.join(rwdir, "cloned_repo") + cloned_repo = git.Repo.clone_from(repo_path, cloned_repo_path) + + cloned_repo.submodule_update(init=True, recursive=True) + @with_rw_directory @_patch_git_config("protocol.file.allow", "always") def test_list_only_valid_submodules(self, rwdir): From 088d090ed3016aca68f3376736bf9f2b18d0fe7e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 12 Jun 2025 02:39:11 -0400 Subject: [PATCH 1187/1392] Run `cat_file.py` fixture without site customizations This fixes a new `TestGit::test_handle_process_output` test failure on Cygwin where a `CoverageWarning` was printed to stderr in the Python interpreter subprocess running the `cat_file.py` fixture. We usually run the test suite with `pytest-cov` enabled. This is configured in `pyproject.toml` to happen by default. `pytest-cov` uses the `coverage` module, but it adds some more functionality. This includes instrumenting subprocesses, which is achieved by installing its `pytest-cov.pth` file into `site-packages` to be run by all Python interpreter instances. This causes interpeters to check for environment variables such as `COV_CORE_SOURCE` and to conditionally initialize `pytest_cov`. For details, see: https://pytest-cov.readthedocs.io/en/latest/subprocess-support.html `coverage` 7.9.0 was recently released. One of the changes is to start issuing a warning if it can't import the C tracer core. See: https://github.com/nedbat/coveragepy/releases/tag/7.9.0 If this warning is issued in the `cat_file.py` subprocess used in `test_handle_process_output`, it causes the test to fail, because the subprocess writes two more lines to its standard error stream, which cause the line count to come out as two more than expected: /cygdrive/d/a/GitPython/GitPython/.venv/lib/python3.9/site-packages/coverage/core.py:96: CoverageWarning: Couldn't import C tracer: No module named 'coverage.tracer' (no-ctracer) warn(f"Couldn't import C tracer: {IMPORT_ERROR}", slug="no-ctracer", once=True) On most platforms, there is no failure, because the condition the warnings describe does not occur, so there are no warnings. But on Cygwin it does occur, resulting in a new test failure, showing > self.assertEqual(len(actual_lines[2]), expected_line_count, repr(actual_lines[2])) E AssertionError: 5004 != 5002 : ["/cygdrive/d/a/GitPython/GitPython/.venv/lib/python3.9/site-packages/coverage/core.py:96: CoverageWarning: Couldn't import C tracer: No module named 'coverage.tracer' (no-ctracer)\n", ' warn(f"Couldn\'t import C tracer: {IMPORT_ERROR}", slug="no-ctracer", once=True)\n', 'From github.com:jantman/gitpython_issue_301\n', ' = [up to date] master -> origin/master\n', ' = [up to date] testcommit1 -> origin/testcommit1\n', ' = [up to date] testcommit10 -> origin/testcommit10\n', ... where the first two elements of the list are from the lines of the warning message, and the others are as expected. (The above is a highly abridged extract, with the `...` at the end standing for many more list items obtained through the `cat_file.py` fixture.) This new failure is triggered specifically by the new `coverage` package version. It is not due to any recent changes in GitPython. It can be observed by rerunning CI checks that have previously passed, or in: https://github.com/EliahKagan/GitPython/actions/runs/15598239952/job/43940156308#step:14:355 There is more than one possible way to fix this, including fixing the underlying condition being warned about on Cygwin, or sanitizing environment variables for the subprocess. The approach taken here instead is based on the idea that the `cat_file.py` fixture is very simple, and that it is conceptually just a standalone Python script that doesn't do anything meant to depend on the current Python environment. Accordingly, this passes the `-S` option to the interpreter for the `cat_file.py` subprocess, so that interpreter refrains from loading the `site` module. This includes, among other simplifying effects, that the subprocess performs no `.pth` customizations. --- test/test_git.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_git.py b/test/test_git.py index 5bcf89bdd..4a54d0d9b 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -773,6 +773,7 @@ def stderr_handler(line): cmdline = [ sys.executable, + "-S", # Keep any `CoverageWarning` messages out of the subprocess stderr. fixture_path("cat_file.py"), str(fixture_path("issue-301_stderr")), ] From 66955cc76a9b33716baa6bdcc575d0446cf9175e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 14 Jun 2025 14:42:31 -0400 Subject: [PATCH 1188/1392] Minor CI clarifications - In the Cygwin CI workflow, move `runs-on` below `strategy`, for greater consistency with other workflows. - In the Cygwin CI jobs, use `pwsh` rather than `bash` for the `git config` command run outside of Cygwin, since `pwsh` is the default shell for such commands, it's the shell the Cygwin setup action uses, and it avoids creating the wrong impression that `bash` is needed. - Use "virtual environment" instead of "virtualenv" in some step names to avoid possible confusion with the `virtualenv` pacakge. - Remove comments in the PyPA package upgrade steps, which are more self-documenting since 727f4e9 (#2043). --- .github/workflows/alpine-test.yml | 3 +-- .github/workflows/cygwin-test.yml | 9 ++++----- .github/workflows/pythonpackage.yml | 1 - 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index a3361798d..ceba11fb8 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -47,13 +47,12 @@ jobs: # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig - - name: Set up virtualenv + - name: Set up virtual environment run: | python -m venv .venv - name: Update PyPA packages run: | - # Get the latest pip, wheel, and prior to Python 3.12, setuptools. . .venv/bin/activate python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 6943db09c..28a41a362 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -7,8 +7,6 @@ permissions: jobs: test: - runs-on: windows-latest - strategy: matrix: selection: [fast, perf] @@ -20,6 +18,8 @@ jobs: fail-fast: false + runs-on: windows-latest + env: CHERE_INVOKING: "1" CYGWIN_NOWINPATH: "1" @@ -32,7 +32,7 @@ jobs: - name: Force LF line endings run: | git config --global core.autocrlf false # Affects the non-Cygwin git. - shell: bash # Use Git Bash instead of Cygwin Bash for this step. + shell: pwsh # Do this outside Cygwin, to affect actions/checkout. - uses: actions/checkout@v4 with: @@ -67,7 +67,7 @@ jobs: # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig - - name: Set up virtualenv + - name: Set up virtual environment run: | python3.9 -m venv --without-pip .venv echo 'BASH_ENV=.venv/bin/activate' >>"$GITHUB_ENV" @@ -78,7 +78,6 @@ jobs: - name: Update PyPA packages run: | - # Get the latest pip, wheel, and prior to Python 3.12, setuptools. python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel - name: Install project and test dependencies diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index c56d45df7..f3d977760 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -71,7 +71,6 @@ jobs: - name: Update PyPA packages run: | - # Get the latest pip, wheel, and prior to Python 3.12, setuptools. python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel - name: Install project and test dependencies From 8e24edfb4688180287c970b023516c2b71946778 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 14 Jun 2025 14:57:40 -0400 Subject: [PATCH 1189/1392] Use *-wheel packages as a better fix for #2004 This installs the `python-pip-wheel`, `python-setuptools-wheel`, and `python-wheel-wheel` packages on Cygwini CI, which provide `.whl` files for `pip`, `setuptools`, and `wheel`. By making those wheels available, this fixes #2004 better than the previous workaround, allowing `ensurepip` to run without the error: Traceback (most recent call last): File "/usr/lib/python3.9/runpy.py", line 188, in _run_module_as_main mod_name, mod_spec, code = _get_module_details(mod_name, _Error) File "/usr/lib/python3.9/runpy.py", line 147, in _get_module_details return _get_module_details(pkg_main_name, error) File "/usr/lib/python3.9/runpy.py", line 111, in _get_module_details __import__(pkg_name) File "/usr/lib/python3.9/ensurepip/__init__.py", line [30](https://github.com/EliahKagan/GitPython/actions/runs/13454947366/job/37596811693#step:10:31), in _SETUPTOOLS_VERSION = _get_most_recent_wheel_version("setuptools") File "/usr/lib/python3.9/ensurepip/__init__.py", line 27, in _get_most_recent_wheel_version return str(max(_wheels[pkg], key=distutils.version.LooseVersion)) ValueError: max() arg is an empty sequence This change takes the place of the main changes in #2007 and #2009. In particular, it should allow `test_installation` to pass again. This also delists non-wheel Cygwin packages such as `python39-pip`, which are not needed (or at least no longer needed). (The python-{pip,setuptools,wheel}-wheel packages are, as their names suggest, intentionally not specific to Python 3.9. However, this technique will not necessarily carry over to Python 3.12, depending on what versions are supplied and other factors. This may be relevant when another attempt like #1988 is made to test/support Python 3.12 on Cygwin. At least for now, though, this seems worthwhile for fixing the Cygwin 3.9 environment, making it more similar to working local Cygwin environments and letting the workflow be more usable as guidance to how to set up a local Cygwin environment for GitPython development, and letting the installation test pass automatically.) --- .github/workflows/cygwin-test.yml | 8 ++------ test/test_installation.py | 8 -------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 28a41a362..2d0378490 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -41,7 +41,7 @@ jobs: - name: Install Cygwin uses: cygwin/cygwin-install-action@v5 with: - packages: python39 python39-pip python39-virtualenv git wget + packages: git python39 python-pip-wheel python-setuptools-wheel python-wheel-wheel add-to-path: false # No need to change $PATH outside the Cygwin environment. - name: Arrange for verbose output @@ -69,13 +69,9 @@ jobs: - name: Set up virtual environment run: | - python3.9 -m venv --without-pip .venv + python3.9 -m venv .venv echo 'BASH_ENV=.venv/bin/activate' >>"$GITHUB_ENV" - - name: Bootstrap pip in virtualenv - run: | - wget -qO- https://bootstrap.pypa.io/get-pip.py | python - - name: Update PyPA packages run: | python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel diff --git a/test/test_installation.py b/test/test_installation.py index a35826bd0..ae6472e98 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -4,19 +4,11 @@ import ast import os import subprocess -import sys - -import pytest from test.lib import TestBase, VirtualEnvironment, with_rw_directory class TestInstallation(TestBase): - @pytest.mark.xfail( - sys.platform == "cygwin" and "CI" in os.environ, - reason="Trouble with pip on Cygwin CI, see issue #2004", - raises=subprocess.CalledProcessError, - ) @with_rw_directory def test_installation(self, rw_dir): venv = self._set_up_venv(rw_dir) From 953d1616e7385ec6aac1e7a6212c689874c9409c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 15 Jun 2025 19:04:15 -0400 Subject: [PATCH 1190/1392] Comment what `TestInstallation._set_up_venv` does So that the meaning of the `venv.sources` accesses in `test_installation` is more readily clear. --- test/test_installation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_installation.py b/test/test_installation.py index ae6472e98..8231b3512 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -64,10 +64,14 @@ def test_installation(self, rw_dir): @staticmethod def _set_up_venv(rw_dir): + # Initialize the virtual environment. venv = VirtualEnvironment(rw_dir, with_pip=True) + + # Make its src directory a symlink to our own top-level source tree. os.symlink( os.path.dirname(os.path.dirname(__file__)), venv.sources, target_is_directory=True, ) + return venv From 84632c78512e070a7aaa9ff1dd046c77293a58a4 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 15 Jun 2025 19:06:58 -0400 Subject: [PATCH 1191/1392] Extract `subprocess.run` logic repeated in `test_installation` This creates a function (technically, a callable `partial` object) for `test_installation` to use instead of repeating `subproces.run` keyword arguments all the time. This relates directly to steps in `_set_up_venv`, and it's makes about as much sense to do it there as in `test_installation`, so it is placed (and described) in `_set_up_venv`. --- test/test_installation.py | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/test/test_installation.py b/test/test_installation.py index 8231b3512..b428f413a 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -2,6 +2,7 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import ast +import functools import os import subprocess @@ -11,35 +12,23 @@ class TestInstallation(TestBase): @with_rw_directory def test_installation(self, rw_dir): - venv = self._set_up_venv(rw_dir) + venv, run = self._set_up_venv(rw_dir) - result = subprocess.run( - [venv.pip, "install", "."], - stdout=subprocess.PIPE, - cwd=venv.sources, - ) + result = run([venv.pip, "install", "."]) self.assertEqual( 0, result.returncode, msg=result.stderr or result.stdout or "Can't install project", ) - result = subprocess.run( - [venv.python, "-c", "import git"], - stdout=subprocess.PIPE, - cwd=venv.sources, - ) + result = run([venv.python, "-c", "import git"]) self.assertEqual( 0, result.returncode, msg=result.stderr or result.stdout or "Self-test failed", ) - result = subprocess.run( - [venv.python, "-c", "import gitdb; import smmap"], - stdout=subprocess.PIPE, - cwd=venv.sources, - ) + result = run([venv.python, "-c", "import gitdb; import smmap"]) self.assertEqual( 0, result.returncode, @@ -49,11 +38,7 @@ def test_installation(self, rw_dir): # Even IF gitdb or any other dependency is supplied during development by # inserting its location into PYTHONPATH or otherwise patched into sys.path, # make sure it is not wrongly inserted as the *first* entry. - result = subprocess.run( - [venv.python, "-c", "import sys; import git; print(sys.path)"], - stdout=subprocess.PIPE, - cwd=venv.sources, - ) + result = run([venv.python, "-c", "import sys; import git; print(sys.path)"]) syspath = result.stdout.decode("utf-8").splitlines()[0] syspath = ast.literal_eval(syspath) self.assertEqual( @@ -74,4 +59,11 @@ def _set_up_venv(rw_dir): target_is_directory=True, ) - return venv + # Create a convenience function to run commands in it. + run = functools.partial( + subprocess.run, + stdout=subprocess.PIPE, + cwd=venv.sources, + ) + + return venv, run From a2ba4804a6544642b400dced6eac0f4f1ad28750 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 15 Jun 2025 19:11:16 -0400 Subject: [PATCH 1192/1392] Have `test_installation` test that operations produce no warnings By setting the `PYTHONWARNINGS` environment variable to `error` in each of the subprocess invocations. This is strictly stronger than passing `-Werror` for the `python` commands, because it automatically applies to subprocesses (unless they are created with a sanitized environment or otherwise with one in which `PYTHONWARNINGS` has been customized), and because it works for `pip` automatically. Importantly, this will cause warnings internal to Python subprocesses created by `pip` to be treated as errors. It should thus surface any warnings coming from the `setuptools` backend. --- test/test_installation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_installation.py b/test/test_installation.py index b428f413a..5ba8a82b0 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -64,6 +64,7 @@ def _set_up_venv(rw_dir): subprocess.run, stdout=subprocess.PIPE, cwd=venv.sources, + env={**os.environ, "PYTHONWARNINGS": "error"}, ) return venv, run From a0e08fe90135149de203a3adfb21a5a254cb1bdb Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 15 Jun 2025 19:48:39 -0400 Subject: [PATCH 1193/1392] Show more information when `test_installation` fails Previously, it attempted to show stderr unless empty, first falling back to stdout unless empty, then falling back to the prewritten summary identifying the specific assertion. This now has the `test_installation` assertions capture stderr as well as stdout, handle standard streams as text rather than binary, and show more information when failing, always distinguishing where the information came from: the summary, then labeled captured stdout (empty or not), then labeled captured stderr (empty or not). That applies to all but the last assertion, which does not try to show information differently when it fails, but is simplified to do the right thing now that `subprocess.run` is using text streams. (This subtly changes its semantics, but overall it should be as effective as before at finding the `sys.path` woe it anticipates.) --- test/test_installation.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/test/test_installation.py b/test/test_installation.py index 5ba8a82b0..39797c134 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -15,31 +15,19 @@ def test_installation(self, rw_dir): venv, run = self._set_up_venv(rw_dir) result = run([venv.pip, "install", "."]) - self.assertEqual( - 0, - result.returncode, - msg=result.stderr or result.stdout or "Can't install project", - ) + self._check_result(result, "Can't install project") result = run([venv.python, "-c", "import git"]) - self.assertEqual( - 0, - result.returncode, - msg=result.stderr or result.stdout or "Self-test failed", - ) + self._check_result(result, "Self-test failed") result = run([venv.python, "-c", "import gitdb; import smmap"]) - self.assertEqual( - 0, - result.returncode, - msg=result.stderr or result.stdout or "Dependencies not installed", - ) + self._check_result(result, "Dependencies not installed") # Even IF gitdb or any other dependency is supplied during development by # inserting its location into PYTHONPATH or otherwise patched into sys.path, # make sure it is not wrongly inserted as the *first* entry. result = run([venv.python, "-c", "import sys; import git; print(sys.path)"]) - syspath = result.stdout.decode("utf-8").splitlines()[0] + syspath = result.stdout.splitlines()[0] syspath = ast.literal_eval(syspath) self.assertEqual( "", @@ -63,8 +51,17 @@ def _set_up_venv(rw_dir): run = functools.partial( subprocess.run, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, cwd=venv.sources, env={**os.environ, "PYTHONWARNINGS": "error"}, ) return venv, run + + def _check_result(self, result, failure_summary): + self.assertEqual( + 0, + result.returncode, + f"{failure_summary}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}", + ) From 6826b594e2ef43b0972f29f335e7be51c9574303 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 15 Jun 2025 20:05:56 -0400 Subject: [PATCH 1194/1392] Improve failure message whitespace clarity --- test/test_installation.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/test_installation.py b/test/test_installation.py index 39797c134..da0b86ed2 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -63,5 +63,11 @@ def _check_result(self, result, failure_summary): self.assertEqual( 0, result.returncode, - f"{failure_summary}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}", + self._prepare_failure_message(result, failure_summary), ) + + @staticmethod + def _prepare_failure_message(result, failure_summary): + stdout = result.stdout.rstrip() + stderr = result.stderr.rstrip() + return f"{failure_summary}\n\nstdout:\n{stdout}\n\nstderr:\n{stderr}" From d0868bd5d6a43c3d95a3eaddab1389a0ef76d8f0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 15 Jun 2025 20:25:55 -0400 Subject: [PATCH 1195/1392] Remove deprecated license classifier GitPython project metadata specify the project's license primarily through the value of `license`, currently passed as an argument to `setuptools.setup`, which holds the string `"BSD-3-Clause"`. This is an SPDX license identifier readily understood by both humans and machines. The PyPI trove classifier "License :: OSI Approved :: BSD License" has also been specified in `setup.py`. However, this is not ideal, because: 1. It does not identify a specific license. There are multiple "BSD" licenses in use, with BSD-2-Clause and BSD-3-Clause both in very wide use. 2. It is no longer recommended to use a trove classifier to indicate a license. The use of license classifiers (even unambiguous ones) has been deprecated. See: - https://packaging.python.org/en/latest/specifications/core-metadata/#classifier-multiple-use - https://peps.python.org/pep-0639/#deprecate-license-classifiers This commit removes the classifier. The license itself is of course unchanged, as is the `license` value of `"BSD-3-Clause"`. (An expected effect of this change is that, starting in the next release of GitPython, PyPI may show "License: BSD-3-Clause" instead of the current text "License: BSD License (BSD-3-Clause)".) This change fixes a warning issued by a subprocess of `pip` when installing the package. The warning, until this change, could be observed by running `pip install . -v` or `pip install -e . -v` and examining the verbose output, or by running `pip install .` or `pip install -e .` with the `PYTHONWARNINGS` environment variable set to `error`: SetuptoolsDeprecationWarning: License classifiers are deprecated. !! ******************************************************************************** Please consider removing the following classifiers in favor of a SPDX license expression: License :: OSI Approved :: BSD License See https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#license for details. ******************************************************************************** !! (In preceding commits, `test_installation` has been augmented to set that environment variable, surfacing the error. This change should allow that test to pass, unless it finds other problems.) --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index f28fedb85..a7b1eab00 100755 --- a/setup.py +++ b/setup.py @@ -95,7 +95,6 @@ def _stamp_version(filename: str) -> None: # "Development Status :: 7 - Inactive", "Environment :: Console", "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Operating System :: POSIX", "Operating System :: Microsoft :: Windows", From b21c32a302dcd91d52636e5f91e89ae129654bd2 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 15 Jun 2025 23:37:27 -0400 Subject: [PATCH 1196/1392] Pass assertion `msg` as a keyword argument in `test_installation` It was originally made explicit like this, but it ended up becoming position in my last few commits. This restores that clearer aspect of how it was written before, while keeping all the other changes. --- test/test_installation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_installation.py b/test/test_installation.py index da0b86ed2..7c82bd403 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -63,7 +63,7 @@ def _check_result(self, result, failure_summary): self.assertEqual( 0, result.returncode, - self._prepare_failure_message(result, failure_summary), + msg=self._prepare_failure_message(result, failure_summary), ) @staticmethod From ac3437df0c5b5ad597915db15276dcb326d3d970 Mon Sep 17 00:00:00 2001 From: Tom Bedor Date: Tue, 17 Jun 2025 09:03:52 -0700 Subject: [PATCH 1197/1392] Add clearer error version for unsupported index error --- git/index/fun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/index/fun.py b/git/index/fun.py index 59cce6ae6..d03ec6759 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -207,7 +207,7 @@ def read_header(stream: IO[bytes]) -> Tuple[int, int]: version, num_entries = unpacked # TODO: Handle version 3: extended data, see read-cache.c. - assert version in (1, 2) + assert version in (1, 2), "Unsupported git index version %i, only 1 and 2 are supported" % version return version, num_entries From bb5c22629aba914a00b8f33a10522324ca7bb049 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 15:20:34 +0000 Subject: [PATCH 1198/1392] Bump Vampire/setup-wsl from 5.0.1 to 6.0.0 Bumps [Vampire/setup-wsl](https://github.com/vampire/setup-wsl) from 5.0.1 to 6.0.0. - [Release notes](https://github.com/vampire/setup-wsl/releases) - [Commits](https://github.com/vampire/setup-wsl/compare/v5.0.1...v6.0.0) --- updated-dependencies: - dependency-name: Vampire/setup-wsl dependency-version: 6.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index f3d977760..4457a341f 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -51,7 +51,7 @@ jobs: - name: Set up WSL (Windows) if: matrix.os-type == 'windows' - uses: Vampire/setup-wsl@v5.0.1 + uses: Vampire/setup-wsl@v6.0.0 with: wsl-version: 1 distribution: Alpine From 496392b9bf781904421cbd171c0c5395a6fe330c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 15:21:27 +0000 Subject: [PATCH 1199/1392] Bump cygwin/cygwin-install-action from 5 to 6 Bumps [cygwin/cygwin-install-action](https://github.com/cygwin/cygwin-install-action) from 5 to 6. - [Release notes](https://github.com/cygwin/cygwin-install-action/releases) - [Commits](https://github.com/cygwin/cygwin-install-action/compare/v5...v6) --- updated-dependencies: - dependency-name: cygwin/cygwin-install-action dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/cygwin-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 2d0378490..cc9e1edf0 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -39,7 +39,7 @@ jobs: fetch-depth: 0 - name: Install Cygwin - uses: cygwin/cygwin-install-action@v5 + uses: cygwin/cygwin-install-action@v6 with: packages: git python39 python-pip-wheel python-setuptools-wheel python-wheel-wheel add-to-path: false # No need to change $PATH outside the Cygwin environment. From a4aadb0c04bd13af824c14dcc39f88345aa5c440 Mon Sep 17 00:00:00 2001 From: Niklas Mertsch Date: Sun, 20 Jul 2025 14:17:49 +0200 Subject: [PATCH 1200/1392] Fix name collision --- git/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/git/config.py b/git/config.py index 5fc099a27..345290a39 100644 --- a/git/config.py +++ b/git/config.py @@ -87,15 +87,15 @@ def __new__(cls, name: str, bases: Tuple, clsdict: Dict[str, Any]) -> "MetaParse mutating_methods = clsdict[kmm] for base in bases: methods = (t for t in inspect.getmembers(base, inspect.isroutine) if not t[0].startswith("_")) - for name, method in methods: - if name in clsdict: + for method_name, method in methods: + if method_name in clsdict: continue method_with_values = needs_values(method) - if name in mutating_methods: + if method_name in mutating_methods: method_with_values = set_dirty_and_flush_changes(method_with_values) # END mutating methods handling - clsdict[name] = method_with_values + clsdict[method_name] = method_with_values # END for each name/method pair # END for each base # END if mutating methods configuration is set From 80fd2c16211738156e65258381a17cdc429ddd08 Mon Sep 17 00:00:00 2001 From: Niklas Mertsch Date: Sun, 20 Jul 2025 14:17:58 +0200 Subject: [PATCH 1201/1392] Don't treat sphinx warnings as errors Workaround for python/cpython#100520 (rst syntax error in configparser docstrings), which was fixed in CPython 3.10+. Docutils raises warnings about the invalid docstrings, and `-W` instructs sphinx to treat this as errors. We can't control or silence these warnings, so we accept them and don't treat them as errors. See the discussion in gitpython-developers/GitPython#2060 for details. --- doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index ddeadbd7e..7e0d325fe 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line. BUILDDIR = build -SPHINXOPTS = -W +SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = From ec2e2c8b894512e7a2364774d77cdd9db73f0566 Mon Sep 17 00:00:00 2001 From: Tom Webber Date: Tue, 22 Jul 2025 12:23:18 +0200 Subject: [PATCH 1202/1392] Allow relative path url in submodules for submodule_update --- git/objects/submodule/base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 0e55b8fa9..5031a2e71 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -11,6 +11,7 @@ import stat import sys import uuid +import urllib import git from git.cmd import Git @@ -799,9 +800,13 @@ def update( + "Cloning url '%s' to '%s' in submodule %r" % (self.url, checkout_module_abspath, self.name), ) if not dry_run: + if self.url.startswith("."): + url = urllib.parse.urljoin(self.repo.remotes.origin.url + "/", self.url) + else: + url = self.url mrepo = self._clone_repo( self.repo, - self.url, + url, self.path, self.name, n=True, From 1ee1e781929074afd66bff1eae007bbee41d117e Mon Sep 17 00:00:00 2001 From: Tom Webber Date: Wed, 23 Jul 2025 07:12:27 +0200 Subject: [PATCH 1203/1392] Add test case for cloning submodules with relative path --- test/test_submodule.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/test_submodule.py b/test/test_submodule.py index f44f086c2..4a248eb60 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -1350,3 +1350,23 @@ def test_submodule_update_unsafe_options_allowed(self, rw_repo): for unsafe_option in unsafe_options: with self.assertRaises(GitCommandError): submodule.update(clone_multi_options=[unsafe_option], allow_unsafe_options=True) + + @with_rw_directory + @_patch_git_config("protocol.file.allow", "always") + def test_submodule_update_relative_url(self, rwdir): + parent_path = osp.join(rwdir, "parent") + parent_repo = git.Repo.init(parent_path) + submodule_path = osp.join(rwdir, "module") + submodule_repo = git.Repo.init(submodule_path) + submodule_repo.git.commit(m="initial commit", allow_empty=True) + + parent_repo.git.submodule("add", "../module", "module") + parent_repo.index.commit("add submodule with relative URL") + + cloned_path = osp.join(rwdir, "cloned_repo") + cloned_repo = git.Repo.clone_from(parent_path, cloned_path) + + cloned_repo.submodule_update(init=True, recursive=True) + + has_module = any(sm.name == "module" for sm in cloned_repo.submodules) + assert has_module, "Relative submodule was not updated properly" From 6ba2c0a2f9ee7feffd7e079621c4845820180c9a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 24 Jul 2025 05:37:18 +0200 Subject: [PATCH 1204/1392] Prepare a new release --- VERSION | 2 +- doc/source/changes.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e6af1c454..3c91929a4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.44 +3.1.45 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 00a3c660e..151059ed2 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ========= +3.1.45 +====== + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.45 + 3.1.44 ====== From de6d177aaa8ccd6d82576ab0b3de5074cf7647fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Aug 2025 18:05:41 +0000 Subject: [PATCH 1205/1392] Bump actions/checkout from 4 to 5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/alpine-test.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/cygwin-test.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index ceba11fb8..a9c29117e 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -26,7 +26,7 @@ jobs: adduser runner docker shell: sh -exo pipefail {0} # Run this as root, not the "runner" user. - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 2bee952af..9191471c3 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -47,7 +47,7 @@ jobs: # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Add any setup steps before running the `github/codeql-action/init` action. # This includes steps like installing compilers or runtimes (`actions/setup-node` diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index cc9e1edf0..5c42c8583 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -34,7 +34,7 @@ jobs: git config --global core.autocrlf false # Affects the non-Cygwin git. shell: pwsh # Do this outside Cygwin, to affect actions/checkout. - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ceba0dd85..16978f9a8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: actions/setup-python@v5 with: diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4457a341f..4e5d82a55 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -39,7 +39,7 @@ jobs: shell: bash --noprofile --norc -exo pipefail {0} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 From fd655f90f14b4fa45cc88da2814f99ee5d6279de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 03:59:37 +0000 Subject: [PATCH 1206/1392] Bump actions/checkout from 4 to 5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 8935881e7..29494a0f7 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -24,7 +24,7 @@ jobs: continue-on-error: ${{ matrix.experimental }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: From 366859fd74ec5dfe36443dcbc7e752383fb689fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 15:02:33 +0000 Subject: [PATCH 1207/1392] Bump actions/checkout from 4 to 5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index c5d7e2b8c..6730e7da5 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -25,7 +25,7 @@ jobs: continue-on-error: ${{ matrix.experimental }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} From 70abd0ee5d4c9c7104c4f5ad009e82b45b71a852 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 16:46:16 +0000 Subject: [PATCH 1208/1392] Bump gitdb/ext/smmap from `c6b53d3` to `1de0797` Bumps [gitdb/ext/smmap](https://github.com/gitpython-developers/smmap) from `c6b53d3` to `1de0797`. - [Release notes](https://github.com/gitpython-developers/smmap/releases) - [Commits](https://github.com/gitpython-developers/smmap/compare/c6b53d35deb82a38d5d07ca7712c1334a7a10c10...1de0797344ed031cc1d5f9024f01e8093b02baa9) --- updated-dependencies: - dependency-name: gitdb/ext/smmap dependency-version: 1de0797344ed031cc1d5f9024f01e8093b02baa9 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- gitdb/ext/smmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index c6b53d35d..1de079734 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit c6b53d35deb82a38d5d07ca7712c1334a7a10c10 +Subproject commit 1de0797344ed031cc1d5f9024f01e8093b02baa9 From fe81519436a8ab8b735a40a3973c8c5bd9cfec47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 20:42:38 +0000 Subject: [PATCH 1209/1392] Bump git/ext/gitdb from `335c0f6` to `39d7dbf` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `335c0f6` to `39d7dbf`. - [Release notes](https://github.com/gitpython-developers/gitdb/releases) - [Commits](https://github.com/gitpython-developers/gitdb/compare/335c0f66173eecdc7b2597c2b6c3d1fde795df30...39d7dbf285df058e44ea501c23ea8d31ae8bce0e) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-version: 39d7dbf285df058e44ea501c23ea8d31ae8bce0e dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 335c0f661..39d7dbf28 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 335c0f66173eecdc7b2597c2b6c3d1fde795df30 +Subproject commit 39d7dbf285df058e44ea501c23ea8d31ae8bce0e From ca51dad69071898af377c8e62210c69e8d211c69 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Sep 2025 14:41:33 +0000 Subject: [PATCH 1210/1392] Bump actions/setup-python from 5 to 6 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/lint.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 16978f9a8..ed535a914 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.x" diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4e5d82a55..7088310e5 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -44,7 +44,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} From 7f39c7473e9799bdee5d08235470f1086ac16f02 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Sep 2025 14:42:01 +0000 Subject: [PATCH 1211/1392] Bump actions/setup-python from 5 to 6 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 6730e7da5..30ebce40a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -29,7 +29,7 @@ jobs: with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} From 7b0bb5ec90e7f9766c66ba2a8b350bb59bee42d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Sep 2025 14:42:42 +0000 Subject: [PATCH 1212/1392] Bump actions/setup-python from 5 to 6 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 29494a0f7..cf64e4ab1 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -26,7 +26,7 @@ jobs: steps: - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} From 707b78545690ff916e5441a93e3e41bba6769ee5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Sep 2025 14:54:24 +0000 Subject: [PATCH 1213/1392] Bump gitdb/ext/smmap from `1de0797` to `801bd6f` Bumps [gitdb/ext/smmap](https://github.com/gitpython-developers/smmap) from `1de0797` to `801bd6f`. - [Release notes](https://github.com/gitpython-developers/smmap/releases) - [Commits](https://github.com/gitpython-developers/smmap/compare/1de0797344ed031cc1d5f9024f01e8093b02baa9...801bd6f5722aa21be54ea5b113b7a73595857e1c) --- updated-dependencies: - dependency-name: gitdb/ext/smmap dependency-version: 801bd6f5722aa21be54ea5b113b7a73595857e1c dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- gitdb/ext/smmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 1de079734..801bd6f57 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 1de0797344ed031cc1d5f9024f01e8093b02baa9 +Subproject commit 801bd6f5722aa21be54ea5b113b7a73595857e1c From 9f913ec0cb0c6f7ab3eea7245657d01048fd7065 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Sep 2025 15:00:47 +0000 Subject: [PATCH 1214/1392] Bump git/ext/gitdb from `39d7dbf` to `f8fdfec` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `39d7dbf` to `f8fdfec`. - [Release notes](https://github.com/gitpython-developers/gitdb/releases) - [Commits](https://github.com/gitpython-developers/gitdb/compare/39d7dbf285df058e44ea501c23ea8d31ae8bce0e...f8fdfec0fd0a0aed9171c6cf2c5cb8d73e2bb305) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-version: f8fdfec0fd0a0aed9171c6cf2c5cb8d73e2bb305 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 39d7dbf28..f8fdfec0f 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 39d7dbf285df058e44ea501c23ea8d31ae8bce0e +Subproject commit f8fdfec0fd0a0aed9171c6cf2c5cb8d73e2bb305 From 7c55a2b839e05f10a9dc3cf2bc53785350372c88 Mon Sep 17 00:00:00 2001 From: Emmanuel Ferdman Date: Tue, 30 Sep 2025 17:47:14 +0300 Subject: [PATCH 1215/1392] Fix type hint for `SymbolicReference.reference` property Signed-off-by: Emmanuel Ferdman --- git/refs/symbolic.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 1b90a3115..74bb1fe0a 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -40,6 +40,7 @@ from git.config import GitConfigParser from git.objects.commit import Actor from git.refs.log import RefLogEntry + from git.refs.reference import Reference from git.repo import Repo @@ -404,7 +405,7 @@ def object(self) -> AnyGitObject: def object(self, object: Union[AnyGitObject, "SymbolicReference", str]) -> "SymbolicReference": return self.set_object(object) - def _get_reference(self) -> "SymbolicReference": + def _get_reference(self) -> "Reference": """ :return: :class:`~git.refs.reference.Reference` object we point to @@ -416,7 +417,7 @@ def _get_reference(self) -> "SymbolicReference": sha, target_ref_path = self._get_ref_info(self.repo, self.path) if target_ref_path is None: raise TypeError("%s is a detached symbolic reference as it points to %r" % (self, sha)) - return self.from_path(self.repo, target_ref_path) + return cast("Reference", self.from_path(self.repo, target_ref_path)) def set_reference( self, @@ -502,7 +503,7 @@ def set_reference( # Aliased reference @property - def reference(self) -> "SymbolicReference": + def reference(self) -> "Reference": return self._get_reference() @reference.setter From bcdcccdc7ea7d50ec5831aad961ba80df0f1379b Mon Sep 17 00:00:00 2001 From: Brunno Vanelli Date: Tue, 7 Oct 2025 20:49:46 +0200 Subject: [PATCH 1216/1392] feat: Add support for hasconfig git rule. --- git/config.py | 8 ++++++-- test/test_config.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/git/config.py b/git/config.py index 345290a39..ffe1c8ccd 100644 --- a/git/config.py +++ b/git/config.py @@ -66,7 +66,7 @@ CONFIG_LEVELS: ConfigLevels_Tup = ("system", "user", "global", "repository") """The configuration level of a configuration file.""" -CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeIf )\"(gitdir|gitdir/i|onbranch):(.+)\"") +CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeIf )\"(gitdir|gitdir/i|onbranch|hasconfig:remote\.\*\.url):(.+)\"") """Section pattern to detect conditional includes. See: https://git-scm.com/docs/git-config#_conditional_includes @@ -590,7 +590,11 @@ def _included_paths(self) -> List[Tuple[str, str]]: if fnmatch.fnmatchcase(branch_name, value): paths += self.items(section) - + elif keyword == "hasconfig:remote.*.url": + for remote in self._repo.remotes: + if fnmatch.fnmatch(remote.url, value): + paths += self.items(section) + break return paths def read(self) -> None: # type: ignore[override] diff --git a/test/test_config.py b/test/test_config.py index 8e1007d9e..56ac0f304 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -373,6 +373,41 @@ def test_conditional_includes_from_branch_name_error(self, rw_dir): assert not config._has_includes() assert config._included_paths() == [] + @with_rw_directory + def test_conditional_includes_remote_url(self, rw_dir): + # Initiate mocked repository. + repo = mock.Mock() + repo.remotes = [mock.Mock(url="https://github.com/foo/repo")] + + # Initiate config files. + path1 = osp.join(rw_dir, "config1") + path2 = osp.join(rw_dir, "config2") + template = '[includeIf "hasconfig:remote.*.url:{}"]\n path={}\n' + + # Ensure that config with hasconfig and full url is correct. + with open(path1, "w") as stream: + stream.write(template.format("https://github.com/foo/repo", path2)) + + with GitConfigParser(path1, repo=repo) as config: + assert config._has_includes() + assert config._included_paths() == [("path", path2)] + + # Ensure that config with hasconfig and incorrect url is incorrect. + with open(path1, "w") as stream: + stream.write(template.format("incorrect", path2)) + + with GitConfigParser(path1, repo=repo) as config: + assert not config._has_includes() + assert config._included_paths() == [] + + # Ensure that config with hasconfig and url using glob pattern is correct. + with open(path1, "w") as stream: + stream.write(template.format("**/**github.com*/**", path2)) + + with GitConfigParser(path1, repo=repo) as config: + assert config._has_includes() + assert config._included_paths() == [("path", path2)] + def test_rename(self): file_obj = self._to_memcache(fixture_path("git_config")) with GitConfigParser(file_obj, read_only=False, merge_includes=False) as cw: From 6cf863374820a1bcf1fa14b3c2ea87214752bf74 Mon Sep 17 00:00:00 2001 From: Brunno Vanelli Date: Tue, 7 Oct 2025 21:19:31 +0200 Subject: [PATCH 1217/1392] fix: Use fnmatch instead. --- git/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/config.py b/git/config.py index ffe1c8ccd..200c81bb7 100644 --- a/git/config.py +++ b/git/config.py @@ -592,7 +592,7 @@ def _included_paths(self) -> List[Tuple[str, str]]: paths += self.items(section) elif keyword == "hasconfig:remote.*.url": for remote in self._repo.remotes: - if fnmatch.fnmatch(remote.url, value): + if fnmatch.fnmatchcase(remote.url, value): paths += self.items(section) break return paths From a6247a585600c09894a9fae85e11f7581bfccbe0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Oct 2025 13:32:58 +0000 Subject: [PATCH 1218/1392] Bump github/codeql-action from 3 to 4 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3 to 4. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v3...v4) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 9191471c3..32d5e84e4 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -57,7 +57,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -85,6 +85,6 @@ jobs: exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 with: category: "/language:${{matrix.language}}" From 9dd0081213d57f41919ed37e93656410277bfb0b Mon Sep 17 00:00:00 2001 From: Andreas Oberritter Date: Tue, 21 Oct 2025 11:11:16 +0200 Subject: [PATCH 1219/1392] Use actual return type in annotation for method submodule_update Fixes #2077 --- git/repo/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/repo/base.py b/git/repo/base.py index 7e918df8c..6ea96aad2 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -520,7 +520,7 @@ def iter_submodules(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: """ return RootModule(self).traverse(*args, **kwargs) - def submodule_update(self, *args: Any, **kwargs: Any) -> Iterator[Submodule]: + def submodule_update(self, *args: Any, **kwargs: Any) -> RootModule: """Update the submodules, keeping the repository consistent as it will take the previous state into consideration. From 8350bd5a434956d70959edeebf4f15f57bbe9157 Mon Sep 17 00:00:00 2001 From: sminux Date: Sat, 1 Nov 2025 09:16:41 +0300 Subject: [PATCH 1220/1392] Update pack.py - SonarQube issues fix #129 --- gitdb/pack.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/gitdb/pack.py b/gitdb/pack.py index e559e113d..2e0707947 100644 --- a/gitdb/pack.py +++ b/gitdb/pack.py @@ -267,8 +267,6 @@ def close(self): def _set_cache_(self, attr): if attr == "_packfile_checksum": self._packfile_checksum = self._cursor.map()[-40:-20] - elif attr == "_packfile_checksum": - self._packfile_checksum = self._cursor.map()[-20:] elif attr == "_cursor": # Note: We don't lock the file when reading as we cannot be sure # that we can actually write to the location - it could be a read-only @@ -848,7 +846,6 @@ def is_valid_stream(self, sha, use_crc=False): assert shawriter.sha(as_hex=False) == sha return shawriter.sha(as_hex=False) == sha # END handle crc/sha verification - return True def info_iter(self): """ From 74ff8e5e1cb814fbf3b916111d7181bd6e3f3906 Mon Sep 17 00:00:00 2001 From: Yikai Zhao Date: Sun, 2 Nov 2025 10:25:33 +0800 Subject: [PATCH 1221/1392] Support index format v3 --- git/index/fun.py | 16 +++++++++++----- git/index/typ.py | 15 ++++++++++++++- test/test_index.py | 25 +++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/git/index/fun.py b/git/index/fun.py index d03ec6759..0b3d79cf1 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -36,7 +36,7 @@ ) from git.util import IndexFileSHA1Writer, finalize_process -from .typ import BaseIndexEntry, IndexEntry, CE_NAMEMASK, CE_STAGESHIFT +from .typ import CE_EXTENDED, BaseIndexEntry, IndexEntry, CE_NAMEMASK, CE_STAGESHIFT from .util import pack, unpack # typing ----------------------------------------------------------------------------- @@ -158,7 +158,7 @@ def write_cache( write = stream_sha.write # Header - version = 2 + version = 3 if any(entry.extended_flags for entry in entries) else 2 write(b"DIRC") write(pack(">LL", version, len(entries))) @@ -172,6 +172,8 @@ def write_cache( plen = len(path) & CE_NAMEMASK # Path length assert plen == len(path), "Path %s too long to fit into index" % entry.path flags = plen | (entry.flags & CE_NAMEMASK_INV) # Clear possible previous values. + if entry.extended_flags: + flags |= CE_EXTENDED write( pack( ">LLLLLL20sH", @@ -185,6 +187,8 @@ def write_cache( flags, ) ) + if entry.extended_flags: + write(pack(">H", entry.extended_flags)) write(path) real_size = (tell() - beginoffset + 8) & ~7 write(b"\0" * ((beginoffset + real_size) - tell())) @@ -206,8 +210,7 @@ def read_header(stream: IO[bytes]) -> Tuple[int, int]: unpacked = cast(Tuple[int, int], unpack(">LL", stream.read(4 * 2))) version, num_entries = unpacked - # TODO: Handle version 3: extended data, see read-cache.c. - assert version in (1, 2), "Unsupported git index version %i, only 1 and 2 are supported" % version + assert version in (1, 2, 3), "Unsupported git index version %i, only 1, 2, and 3 are supported" % version return version, num_entries @@ -260,12 +263,15 @@ def read_cache( ctime = unpack(">8s", read(8))[0] mtime = unpack(">8s", read(8))[0] (dev, ino, mode, uid, gid, size, sha, flags) = unpack(">LLLLLL20sH", read(20 + 4 * 6 + 2)) + extended_flags = 0 + if flags & CE_EXTENDED: + extended_flags = unpack(">H", read(2))[0] path_size = flags & CE_NAMEMASK path = read(path_size).decode(defenc) real_size = (tell() - beginoffset + 8) & ~7 read((beginoffset + real_size) - tell()) - entry = IndexEntry((mode, sha, flags, path, ctime, mtime, dev, ino, uid, gid, size)) + entry = IndexEntry((mode, sha, flags, path, ctime, mtime, dev, ino, uid, gid, size, extended_flags)) # entry_key would be the method to use, but we save the effort. entries[(path, entry.stage)] = entry count += 1 diff --git a/git/index/typ.py b/git/index/typ.py index 974252528..4bcb604ab 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -32,6 +32,9 @@ CE_VALID = 0x8000 CE_STAGESHIFT = 12 +CE_EXT_SKIP_WORKTREE = 0x4000 +CE_EXT_INTENT_TO_ADD = 0x2000 + # } END invariants @@ -87,6 +90,8 @@ class BaseIndexEntryHelper(NamedTuple): uid: int = 0 gid: int = 0 size: int = 0 + # version 3 extended flags, only when (flags & CE_EXTENDED) is set + extended_flags: int = 0 class BaseIndexEntry(BaseIndexEntryHelper): @@ -102,7 +107,7 @@ def __new__( cls, inp_tuple: Union[ Tuple[int, bytes, int, PathLike], - Tuple[int, bytes, int, PathLike, bytes, bytes, int, int, int, int, int], + Tuple[int, bytes, int, PathLike, bytes, bytes, int, int, int, int, int, int], ], ) -> "BaseIndexEntry": """Override ``__new__`` to allow construction from a tuple for backwards @@ -134,6 +139,14 @@ def stage(self) -> int: """ return (self.flags & CE_STAGEMASK) >> CE_STAGESHIFT + @property + def skip_worktree(self) -> bool: + return (self.extended_flags & CE_EXT_SKIP_WORKTREE) > 0 + + @property + def intent_to_add(self) -> bool: + return (self.extended_flags & CE_EXT_INTENT_TO_ADD) > 0 + @classmethod def from_blob(cls, blob: Blob, stage: int = 0) -> "BaseIndexEntry": """:return: Fully equipped BaseIndexEntry at the given stage""" diff --git a/test/test_index.py b/test/test_index.py index cf3b90fa6..6d90d7965 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1218,6 +1218,31 @@ def test_index_add_non_normalized_path(self, rw_repo): rw_repo.index.add(non_normalized_path) + @with_rw_directory + def test_index_version_v3(self, tmp_dir): + tmp_dir = Path(tmp_dir) + with cwd(tmp_dir): + subprocess.run(["git", "init", "-q"], check=True) + file = tmp_dir / "file.txt" + file.write_text("hello") + subprocess.run(["git", "add", "-N", "file.txt"], check=True) + + repo = Repo(tmp_dir) + + assert len(repo.index.entries) == 1 + entry = list(repo.index.entries.values())[0] + assert entry.path == "file.txt" + assert entry.intent_to_add + + file2 = tmp_dir / "file2.txt" + file2.write_text("world") + repo.index.add(["file2.txt"]) + repo.index.write() + + status_str = subprocess.check_output(["git", "status", "--porcelain"], text=True) + assert " A file.txt\n" in status_str + assert "A file2.txt\n" in status_str + class TestIndexUtils: @pytest.mark.parametrize("file_path_type", [str, Path]) From 3150ebdaa43df5be2c27e717807381724131b128 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Nov 2025 13:08:06 +0000 Subject: [PATCH 1222/1392] Bump git/ext/gitdb from `f8fdfec` to `65321a2` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `f8fdfec` to `65321a2`. - [Release notes](https://github.com/gitpython-developers/gitdb/releases) - [Commits](https://github.com/gitpython-developers/gitdb/compare/f8fdfec0fd0a0aed9171c6cf2c5cb8d73e2bb305...65321a28b586df60b9d1508228e2f53a35f938eb) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-version: 65321a28b586df60b9d1508228e2f53a35f938eb dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index f8fdfec0f..65321a28b 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit f8fdfec0fd0a0aed9171c6cf2c5cb8d73e2bb305 +Subproject commit 65321a28b586df60b9d1508228e2f53a35f938eb From 8a884fea3ff91f1444a36785cc22c8d7fc6bf329 Mon Sep 17 00:00:00 2001 From: Yikai Zhao Date: Sat, 8 Nov 2025 14:01:41 +0800 Subject: [PATCH 1223/1392] improve unit test --- test/fixtures/index_extended_flags | Bin 0 -> 436 bytes test/test_index.py | 37 +++++++++++++++++++++-------- 2 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 test/fixtures/index_extended_flags diff --git a/test/fixtures/index_extended_flags b/test/fixtures/index_extended_flags new file mode 100644 index 0000000000000000000000000000000000000000..f03713b684711d4a5aedab0bafd6b254137c9e5d GIT binary patch literal 436 zcmZ?q402{*U|l!>^Ob+tC2sPz z@}frL4{5V^!oJTI8&x~7IWT1AWtQlbFff4htEWE|gV7LkPHLi=!|-hGquHA-UUT;D z)?8N}b>q;Jp5TcNoDK}drAhjUDJiKbK+8Y?WR7Zr=1~|8b=Nnd%;P~auOvSoVcry8 zhad+>tlD0`4JmGI|5U?^ V?tRm?Z1&1ue!#pbGJMwrUjTmQph*A# literal 0 HcmV?d00001 diff --git a/test/test_index.py b/test/test_index.py index 6d90d7965..bb05d3108 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1218,30 +1218,47 @@ def test_index_add_non_normalized_path(self, rw_repo): rw_repo.index.add(non_normalized_path) + def test_index_file_v3(self): + index = IndexFile(self.rorepo, fixture_path("index_extended_flags")) + assert index.entries + assert index.version == 3 + assert len(index.entries) == 4 + assert index.entries[('init.t', 0)].skip_worktree + + # Write the data - it must match the original. + with tempfile.NamedTemporaryFile() as tmpfile: + index.write(tmpfile.name) + assert Path(tmpfile.name).read_bytes() == Path(fixture_path("index_extended_flags")).read_bytes() + @with_rw_directory - def test_index_version_v3(self, tmp_dir): + def test_index_file_v3_with_git_command(self, tmp_dir): tmp_dir = Path(tmp_dir) with cwd(tmp_dir): - subprocess.run(["git", "init", "-q"], check=True) + git = Git(tmp_dir) + git.init() + file = tmp_dir / "file.txt" file.write_text("hello") - subprocess.run(["git", "add", "-N", "file.txt"], check=True) + git.add("--intent-to-add", "file.txt") # intent-to-add sets extended flag repo = Repo(tmp_dir) + index = repo.index - assert len(repo.index.entries) == 1 - entry = list(repo.index.entries.values())[0] + assert len(index.entries) == 1 + assert index.version == 3 + entry = list(index.entries.values())[0] assert entry.path == "file.txt" assert entry.intent_to_add file2 = tmp_dir / "file2.txt" file2.write_text("world") - repo.index.add(["file2.txt"]) - repo.index.write() + index.add(["file2.txt"]) + index.write() - status_str = subprocess.check_output(["git", "status", "--porcelain"], text=True) - assert " A file.txt\n" in status_str - assert "A file2.txt\n" in status_str + status_str = git.status(porcelain=True) + status_lines = status_str.splitlines() + assert " A file.txt" in status_lines + assert "A file2.txt" in status_lines class TestIndexUtils: From 107b1b44e91a19ebbe5e41cd7312ce7838534732 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 9 Nov 2025 08:41:10 +0100 Subject: [PATCH 1224/1392] make linter happy --- test/test_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_index.py b/test/test_index.py index bb05d3108..711b43a0b 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1223,7 +1223,7 @@ def test_index_file_v3(self): assert index.entries assert index.version == 3 assert len(index.entries) == 4 - assert index.entries[('init.t', 0)].skip_worktree + assert index.entries[("init.t", 0)].skip_worktree # Write the data - it must match the original. with tempfile.NamedTemporaryFile() as tmpfile: From 164c34ab47c57e4163a577b3cdb5067ea402cf97 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 02:07:03 +0000 Subject: [PATCH 1225/1392] Bump actions/checkout from 5 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index cf64e4ab1..c08f5b5db 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -24,7 +24,7 @@ jobs: continue-on-error: ${{ matrix.experimental }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: From b5a9cf80d3ca5d7067e723eb3a475df616619748 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 10:56:50 +0000 Subject: [PATCH 1226/1392] Bump gitdb/ext/smmap from `801bd6f` to `5ec977a` Bumps [gitdb/ext/smmap](https://github.com/gitpython-developers/smmap) from `801bd6f` to `5ec977a`. - [Release notes](https://github.com/gitpython-developers/smmap/releases) - [Commits](https://github.com/gitpython-developers/smmap/compare/801bd6f5722aa21be54ea5b113b7a73595857e1c...5ec977a3b280e5dccb40cb20eba56ea26a84bd48) --- updated-dependencies: - dependency-name: gitdb/ext/smmap dependency-version: 5ec977a3b280e5dccb40cb20eba56ea26a84bd48 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- gitdb/ext/smmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 801bd6f57..5ec977a3b 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 801bd6f5722aa21be54ea5b113b7a73595857e1c +Subproject commit 5ec977a3b280e5dccb40cb20eba56ea26a84bd48 From df9d041d6e753fd1b5f21ef7d4c994163e192127 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 11:06:18 +0000 Subject: [PATCH 1227/1392] Bump actions/checkout from 5 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 30ebce40a..ca5ae25d0 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -25,7 +25,7 @@ jobs: continue-on-error: ${{ matrix.experimental }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} From 98e860d1c3a0855e2ff29bf24c5adaca7a57366f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 13:52:55 +0000 Subject: [PATCH 1228/1392] Bump actions/checkout from 5 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/alpine-test.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/cygwin-test.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index a9c29117e..b7de7482e 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -26,7 +26,7 @@ jobs: adduser runner docker shell: sh -exo pipefail {0} # Run this as root, not the "runner" user. - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 32d5e84e4..e243416a8 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -47,7 +47,7 @@ jobs: # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 # Add any setup steps before running the `github/codeql-action/init` action. # This includes steps like installing compilers or runtimes (`actions/setup-node` diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 5c42c8583..327e1f10c 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -34,7 +34,7 @@ jobs: git config --global core.autocrlf false # Affects the non-Cygwin git. shell: pwsh # Do this outside Cygwin, to affect actions/checkout. - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ed535a914..956b38963 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 7088310e5..975c2e29d 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -39,7 +39,7 @@ jobs: shell: bash --noprofile --norc -exo pipefail {0} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 From e3f38ffe1c3e37ec5a8f18d60961ddd0cd0f62f4 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Thu, 27 Nov 2025 17:32:48 +0000 Subject: [PATCH 1229/1392] Move clone tests into dedicated file --- test/test_clone.py | 294 ++++++++++++++++++++++++++++++++++++++++++++- test/test_repo.py | 283 +------------------------------------------ 2 files changed, 294 insertions(+), 283 deletions(-) diff --git a/test/test_clone.py b/test/test_clone.py index 126ef0063..91b7d7621 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -1,12 +1,23 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import os +import os.path as osp +import pathlib +import sys +import tempfile +from unittest import skip + +from git import GitCommandError, Repo +from git.exc import UnsafeOptionError, UnsafeProtocolError + +from test.lib import TestBase, with_rw_directory, with_rw_repo + from pathlib import Path import re import git - -from test.lib import TestBase, with_rw_directory +import pytest class TestClone(TestBase): @@ -29,3 +40,282 @@ def test_checkout_in_non_empty_dir(self, rw_dir): ) else: self.fail("GitCommandError not raised") + + @with_rw_directory + def test_clone_from_pathlib(self, rw_dir): + original_repo = Repo.init(osp.join(rw_dir, "repo")) + + Repo.clone_from(original_repo.git_dir, pathlib.Path(rw_dir) / "clone_pathlib") + + @with_rw_directory + def test_clone_from_pathlib_withConfig(self, rw_dir): + original_repo = Repo.init(osp.join(rw_dir, "repo")) + + cloned = Repo.clone_from( + original_repo.git_dir, + pathlib.Path(rw_dir) / "clone_pathlib_withConfig", + multi_options=[ + "--recurse-submodules=repo", + "--config core.filemode=false", + "--config submodule.repo.update=checkout", + "--config filter.lfs.clean='git-lfs clean -- %f'", + ], + allow_unsafe_options=True, + ) + + self.assertEqual(cloned.config_reader().get_value("submodule", "active"), "repo") + self.assertEqual(cloned.config_reader().get_value("core", "filemode"), False) + self.assertEqual(cloned.config_reader().get_value('submodule "repo"', "update"), "checkout") + self.assertEqual( + cloned.config_reader().get_value('filter "lfs"', "clean"), + "git-lfs clean -- %f", + ) + + def test_clone_from_with_path_contains_unicode(self): + with tempfile.TemporaryDirectory() as tmpdir: + unicode_dir_name = "\u0394" + path_with_unicode = os.path.join(tmpdir, unicode_dir_name) + os.makedirs(path_with_unicode) + + try: + Repo.clone_from( + url=self._small_repo_url(), + to_path=path_with_unicode, + ) + except UnicodeEncodeError: + self.fail("Raised UnicodeEncodeError") + + @with_rw_directory + @skip( + """The referenced repository was removed, and one needs to set up a new + password controlled repo under the org's control.""" + ) + def test_leaking_password_in_clone_logs(self, rw_dir): + password = "fakepassword1234" + try: + Repo.clone_from( + url="https://fakeuser:{}@fakerepo.example.com/testrepo".format(password), + to_path=rw_dir, + ) + except GitCommandError as err: + assert password not in str(err), "The error message '%s' should not contain the password" % err + # Working example from a blank private project. + Repo.clone_from( + url="https://gitlab+deploy-token-392045:mLWhVus7bjLsy8xj8q2V@gitlab.com/mercierm/test_git_python", + to_path=rw_dir, + ) + + @with_rw_repo("HEAD") + def test_clone_unsafe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upload-pack='touch {tmp_file}'", + f"-u 'touch {tmp_file}'", + "--config=protocol.ext.allow=always", + "-c protocol.ext.allow=always", + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + rw_repo.clone(tmp_dir, multi_options=[unsafe_option]) + assert not tmp_file.exists() + + unsafe_options = [ + {"upload-pack": f"touch {tmp_file}"}, + {"u": f"touch {tmp_file}"}, + {"config": "protocol.ext.allow=always"}, + {"c": "protocol.ext.allow=always"}, + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + rw_repo.clone(tmp_dir, **unsafe_option) + assert not tmp_file.exists() + + @pytest.mark.xfail( + sys.platform == "win32", + reason=( + "File not created. A separate Windows command may be needed. This and the " + "currently passing test test_clone_unsafe_options must be adjusted in the " + "same way. Until then, test_clone_unsafe_options is unreliable on Windows." + ), + raises=AssertionError, + ) + @with_rw_repo("HEAD") + def test_clone_unsafe_options_allowed(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upload-pack='touch {tmp_file}'", + f"-u 'touch {tmp_file}'", + ] + for i, unsafe_option in enumerate(unsafe_options): + destination = tmp_dir / str(i) + assert not tmp_file.exists() + # The options will be allowed, but the command will fail. + with self.assertRaises(GitCommandError): + rw_repo.clone(destination, multi_options=[unsafe_option], allow_unsafe_options=True) + assert tmp_file.exists() + tmp_file.unlink() + + unsafe_options = [ + "--config=protocol.ext.allow=always", + "-c protocol.ext.allow=always", + ] + for i, unsafe_option in enumerate(unsafe_options): + destination = tmp_dir / str(i) + assert not destination.exists() + rw_repo.clone(destination, multi_options=[unsafe_option], allow_unsafe_options=True) + assert destination.exists() + + @with_rw_repo("HEAD") + def test_clone_safe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + options = [ + "--depth=1", + "--single-branch", + "-q", + ] + for option in options: + destination = tmp_dir / option + assert not destination.exists() + rw_repo.clone(destination, multi_options=[option]) + assert destination.exists() + + @with_rw_repo("HEAD") + def test_clone_from_unsafe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upload-pack='touch {tmp_file}'", + f"-u 'touch {tmp_file}'", + "--config=protocol.ext.allow=always", + "-c protocol.ext.allow=always", + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + Repo.clone_from(rw_repo.working_dir, tmp_dir, multi_options=[unsafe_option]) + assert not tmp_file.exists() + + unsafe_options = [ + {"upload-pack": f"touch {tmp_file}"}, + {"u": f"touch {tmp_file}"}, + {"config": "protocol.ext.allow=always"}, + {"c": "protocol.ext.allow=always"}, + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + Repo.clone_from(rw_repo.working_dir, tmp_dir, **unsafe_option) + assert not tmp_file.exists() + + @pytest.mark.xfail( + sys.platform == "win32", + reason=( + "File not created. A separate Windows command may be needed. This and the " + "currently passing test test_clone_from_unsafe_options must be adjusted in the " + "same way. Until then, test_clone_from_unsafe_options is unreliable on Windows." + ), + raises=AssertionError, + ) + @with_rw_repo("HEAD") + def test_clone_from_unsafe_options_allowed(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upload-pack='touch {tmp_file}'", + f"-u 'touch {tmp_file}'", + ] + for i, unsafe_option in enumerate(unsafe_options): + destination = tmp_dir / str(i) + assert not tmp_file.exists() + # The options will be allowed, but the command will fail. + with self.assertRaises(GitCommandError): + Repo.clone_from( + rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True + ) + assert tmp_file.exists() + tmp_file.unlink() + + unsafe_options = [ + "--config=protocol.ext.allow=always", + "-c protocol.ext.allow=always", + ] + for i, unsafe_option in enumerate(unsafe_options): + destination = tmp_dir / str(i) + assert not destination.exists() + Repo.clone_from( + rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True + ) + assert destination.exists() + + @with_rw_repo("HEAD") + def test_clone_from_safe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + options = [ + "--depth=1", + "--single-branch", + "-q", + ] + for option in options: + destination = tmp_dir / option + assert not destination.exists() + Repo.clone_from(rw_repo.common_dir, destination, multi_options=[option]) + assert destination.exists() + + def test_clone_from_unsafe_protocol(self): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + urls = [ + f"ext::sh -c touch% {tmp_file}", + "fd::17/foo", + ] + for url in urls: + with self.assertRaises(UnsafeProtocolError): + Repo.clone_from(url, tmp_dir / "repo") + assert not tmp_file.exists() + + def test_clone_from_unsafe_protocol_allowed(self): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + urls = [ + f"ext::sh -c touch% {tmp_file}", + "fd::/foo", + ] + for url in urls: + # The URL will be allowed into the command, but the command will + # fail since we don't have that protocol enabled in the Git config file. + with self.assertRaises(GitCommandError): + Repo.clone_from(url, tmp_dir / "repo", allow_unsafe_protocols=True) + assert not tmp_file.exists() + + def test_clone_from_unsafe_protocol_allowed_and_enabled(self): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + urls = [ + f"ext::sh -c touch% {tmp_file}", + ] + allow_ext = [ + "--config=protocol.ext.allow=always", + ] + for url in urls: + # The URL will be allowed into the command, and the protocol is enabled, + # but the command will fail since it can't read from the remote repo. + assert not tmp_file.exists() + with self.assertRaises(GitCommandError): + Repo.clone_from( + url, + tmp_dir / "repo", + multi_options=allow_ext, + allow_unsafe_protocols=True, + allow_unsafe_options=True, + ) + assert tmp_file.exists() + tmp_file.unlink() diff --git a/test/test_repo.py b/test/test_repo.py index bfa1bbb78..dc2cfe7b1 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -14,7 +14,7 @@ import pickle import sys import tempfile -from unittest import mock, skip +from unittest import mock import pytest @@ -36,7 +36,7 @@ Submodule, Tree, ) -from git.exc import BadObject, UnsafeOptionError, UnsafeProtocolError +from git.exc import BadObject from git.repo.fun import touch from git.util import bin_to_hex, cwd, cygpath, join_path_native, rmfile, rmtree @@ -214,285 +214,6 @@ def test_date_format(self, rw_dir): # @-timestamp is the format used by git commit hooks. repo.index.commit("Commit messages", commit_date="@1400000000 +0000") - @with_rw_directory - def test_clone_from_pathlib(self, rw_dir): - original_repo = Repo.init(osp.join(rw_dir, "repo")) - - Repo.clone_from(original_repo.git_dir, pathlib.Path(rw_dir) / "clone_pathlib") - - @with_rw_directory - def test_clone_from_pathlib_withConfig(self, rw_dir): - original_repo = Repo.init(osp.join(rw_dir, "repo")) - - cloned = Repo.clone_from( - original_repo.git_dir, - pathlib.Path(rw_dir) / "clone_pathlib_withConfig", - multi_options=[ - "--recurse-submodules=repo", - "--config core.filemode=false", - "--config submodule.repo.update=checkout", - "--config filter.lfs.clean='git-lfs clean -- %f'", - ], - allow_unsafe_options=True, - ) - - self.assertEqual(cloned.config_reader().get_value("submodule", "active"), "repo") - self.assertEqual(cloned.config_reader().get_value("core", "filemode"), False) - self.assertEqual(cloned.config_reader().get_value('submodule "repo"', "update"), "checkout") - self.assertEqual( - cloned.config_reader().get_value('filter "lfs"', "clean"), - "git-lfs clean -- %f", - ) - - def test_clone_from_with_path_contains_unicode(self): - with tempfile.TemporaryDirectory() as tmpdir: - unicode_dir_name = "\u0394" - path_with_unicode = os.path.join(tmpdir, unicode_dir_name) - os.makedirs(path_with_unicode) - - try: - Repo.clone_from( - url=self._small_repo_url(), - to_path=path_with_unicode, - ) - except UnicodeEncodeError: - self.fail("Raised UnicodeEncodeError") - - @with_rw_directory - @skip( - """The referenced repository was removed, and one needs to set up a new - password controlled repo under the org's control.""" - ) - def test_leaking_password_in_clone_logs(self, rw_dir): - password = "fakepassword1234" - try: - Repo.clone_from( - url="https://fakeuser:{}@fakerepo.example.com/testrepo".format(password), - to_path=rw_dir, - ) - except GitCommandError as err: - assert password not in str(err), "The error message '%s' should not contain the password" % err - # Working example from a blank private project. - Repo.clone_from( - url="https://gitlab+deploy-token-392045:mLWhVus7bjLsy8xj8q2V@gitlab.com/mercierm/test_git_python", - to_path=rw_dir, - ) - - @with_rw_repo("HEAD") - def test_clone_unsafe_options(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - unsafe_options = [ - f"--upload-pack='touch {tmp_file}'", - f"-u 'touch {tmp_file}'", - "--config=protocol.ext.allow=always", - "-c protocol.ext.allow=always", - ] - for unsafe_option in unsafe_options: - with self.assertRaises(UnsafeOptionError): - rw_repo.clone(tmp_dir, multi_options=[unsafe_option]) - assert not tmp_file.exists() - - unsafe_options = [ - {"upload-pack": f"touch {tmp_file}"}, - {"u": f"touch {tmp_file}"}, - {"config": "protocol.ext.allow=always"}, - {"c": "protocol.ext.allow=always"}, - ] - for unsafe_option in unsafe_options: - with self.assertRaises(UnsafeOptionError): - rw_repo.clone(tmp_dir, **unsafe_option) - assert not tmp_file.exists() - - @pytest.mark.xfail( - sys.platform == "win32", - reason=( - "File not created. A separate Windows command may be needed. This and the " - "currently passing test test_clone_unsafe_options must be adjusted in the " - "same way. Until then, test_clone_unsafe_options is unreliable on Windows." - ), - raises=AssertionError, - ) - @with_rw_repo("HEAD") - def test_clone_unsafe_options_allowed(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - unsafe_options = [ - f"--upload-pack='touch {tmp_file}'", - f"-u 'touch {tmp_file}'", - ] - for i, unsafe_option in enumerate(unsafe_options): - destination = tmp_dir / str(i) - assert not tmp_file.exists() - # The options will be allowed, but the command will fail. - with self.assertRaises(GitCommandError): - rw_repo.clone(destination, multi_options=[unsafe_option], allow_unsafe_options=True) - assert tmp_file.exists() - tmp_file.unlink() - - unsafe_options = [ - "--config=protocol.ext.allow=always", - "-c protocol.ext.allow=always", - ] - for i, unsafe_option in enumerate(unsafe_options): - destination = tmp_dir / str(i) - assert not destination.exists() - rw_repo.clone(destination, multi_options=[unsafe_option], allow_unsafe_options=True) - assert destination.exists() - - @with_rw_repo("HEAD") - def test_clone_safe_options(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - options = [ - "--depth=1", - "--single-branch", - "-q", - ] - for option in options: - destination = tmp_dir / option - assert not destination.exists() - rw_repo.clone(destination, multi_options=[option]) - assert destination.exists() - - @with_rw_repo("HEAD") - def test_clone_from_unsafe_options(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - unsafe_options = [ - f"--upload-pack='touch {tmp_file}'", - f"-u 'touch {tmp_file}'", - "--config=protocol.ext.allow=always", - "-c protocol.ext.allow=always", - ] - for unsafe_option in unsafe_options: - with self.assertRaises(UnsafeOptionError): - Repo.clone_from(rw_repo.working_dir, tmp_dir, multi_options=[unsafe_option]) - assert not tmp_file.exists() - - unsafe_options = [ - {"upload-pack": f"touch {tmp_file}"}, - {"u": f"touch {tmp_file}"}, - {"config": "protocol.ext.allow=always"}, - {"c": "protocol.ext.allow=always"}, - ] - for unsafe_option in unsafe_options: - with self.assertRaises(UnsafeOptionError): - Repo.clone_from(rw_repo.working_dir, tmp_dir, **unsafe_option) - assert not tmp_file.exists() - - @pytest.mark.xfail( - sys.platform == "win32", - reason=( - "File not created. A separate Windows command may be needed. This and the " - "currently passing test test_clone_from_unsafe_options must be adjusted in the " - "same way. Until then, test_clone_from_unsafe_options is unreliable on Windows." - ), - raises=AssertionError, - ) - @with_rw_repo("HEAD") - def test_clone_from_unsafe_options_allowed(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - unsafe_options = [ - f"--upload-pack='touch {tmp_file}'", - f"-u 'touch {tmp_file}'", - ] - for i, unsafe_option in enumerate(unsafe_options): - destination = tmp_dir / str(i) - assert not tmp_file.exists() - # The options will be allowed, but the command will fail. - with self.assertRaises(GitCommandError): - Repo.clone_from( - rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True - ) - assert tmp_file.exists() - tmp_file.unlink() - - unsafe_options = [ - "--config=protocol.ext.allow=always", - "-c protocol.ext.allow=always", - ] - for i, unsafe_option in enumerate(unsafe_options): - destination = tmp_dir / str(i) - assert not destination.exists() - Repo.clone_from( - rw_repo.working_dir, destination, multi_options=[unsafe_option], allow_unsafe_options=True - ) - assert destination.exists() - - @with_rw_repo("HEAD") - def test_clone_from_safe_options(self, rw_repo): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - options = [ - "--depth=1", - "--single-branch", - "-q", - ] - for option in options: - destination = tmp_dir / option - assert not destination.exists() - Repo.clone_from(rw_repo.common_dir, destination, multi_options=[option]) - assert destination.exists() - - def test_clone_from_unsafe_protocol(self): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - urls = [ - f"ext::sh -c touch% {tmp_file}", - "fd::17/foo", - ] - for url in urls: - with self.assertRaises(UnsafeProtocolError): - Repo.clone_from(url, tmp_dir / "repo") - assert not tmp_file.exists() - - def test_clone_from_unsafe_protocol_allowed(self): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - urls = [ - f"ext::sh -c touch% {tmp_file}", - "fd::/foo", - ] - for url in urls: - # The URL will be allowed into the command, but the command will - # fail since we don't have that protocol enabled in the Git config file. - with self.assertRaises(GitCommandError): - Repo.clone_from(url, tmp_dir / "repo", allow_unsafe_protocols=True) - assert not tmp_file.exists() - - def test_clone_from_unsafe_protocol_allowed_and_enabled(self): - with tempfile.TemporaryDirectory() as tdir: - tmp_dir = pathlib.Path(tdir) - tmp_file = tmp_dir / "pwn" - urls = [ - f"ext::sh -c touch% {tmp_file}", - ] - allow_ext = [ - "--config=protocol.ext.allow=always", - ] - for url in urls: - # The URL will be allowed into the command, and the protocol is enabled, - # but the command will fail since it can't read from the remote repo. - assert not tmp_file.exists() - with self.assertRaises(GitCommandError): - Repo.clone_from( - url, - tmp_dir / "repo", - multi_options=allow_ext, - allow_unsafe_protocols=True, - allow_unsafe_options=True, - ) - assert tmp_file.exists() - tmp_file.unlink() - @with_rw_repo("HEAD") def test_max_chunk_size(self, repo): class TestOutputStream(TestBase): From 24abf10dc2913f9c1674c6d60dd70c0ec775a6d4 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Thu, 27 Nov 2025 17:39:15 +0000 Subject: [PATCH 1230/1392] Allow Pathlike urls and destinations when cloning --- git/repo/base.py | 6 ++++-- test/test_clone.py | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index 6ea96aad2..fbed6e471 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -1362,8 +1362,10 @@ def _clone( odbt = kwargs.pop("odbt", odb_default_type) # When pathlib.Path or other class-based path is passed + if not isinstance(url, str): + url = url.__fspath__() if not isinstance(path, str): - path = str(path) + path = path.__fspath__() ## A bug win cygwin's Git, when `--bare` or `--separate-git-dir` # it prepends the cwd or(?) the `url` into the `path, so:: @@ -1380,7 +1382,7 @@ def _clone( multi = shlex.split(" ".join(multi_options)) if not allow_unsafe_protocols: - Git.check_unsafe_protocols(str(url)) + Git.check_unsafe_protocols(url) if not allow_unsafe_options: Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=cls.unsafe_git_clone_options) if not allow_unsafe_options and multi_options: diff --git a/test/test_clone.py b/test/test_clone.py index 91b7d7621..489931458 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -1,6 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +from dataclasses import dataclass import os import os.path as osp import pathlib @@ -45,7 +46,20 @@ def test_checkout_in_non_empty_dir(self, rw_dir): def test_clone_from_pathlib(self, rw_dir): original_repo = Repo.init(osp.join(rw_dir, "repo")) - Repo.clone_from(original_repo.git_dir, pathlib.Path(rw_dir) / "clone_pathlib") + Repo.clone_from(pathlib.Path(original_repo.git_dir), pathlib.Path(rw_dir) / "clone_pathlib") + + @with_rw_directory + def test_clone_from_pathlike(self, rw_dir): + original_repo = Repo.init(osp.join(rw_dir, "repo")) + + @dataclass + class PathLikeMock: + path: str + + def __fspath__(self) -> str: + return self.path + + Repo.clone_from(PathLikeMock(original_repo.git_dir), PathLikeMock(os.path.join(rw_dir, "clone_pathlike"))) @with_rw_directory def test_clone_from_pathlib_withConfig(self, rw_dir): From ad1ae5fea338d2a716506f46532932dc458f791a Mon Sep 17 00:00:00 2001 From: George Ogden Date: Thu, 27 Nov 2025 17:41:07 +0000 Subject: [PATCH 1231/1392] Simplify logic with direct path conversion --- git/index/typ.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/index/typ.py b/git/index/typ.py index 4bcb604ab..5fb1b9abc 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -58,9 +58,9 @@ def __init__(self, paths: Sequence[PathLike]) -> None: def __call__(self, stage_blob: Tuple[StageType, Blob]) -> bool: blob_pathlike: PathLike = stage_blob[1].path - blob_path: Path = blob_pathlike if isinstance(blob_pathlike, Path) else Path(blob_pathlike) + blob_path = Path(blob_pathlike) for pathlike in self.paths: - path: Path = pathlike if isinstance(pathlike, Path) else Path(pathlike) + path = Path(pathlike) # TODO: Change to use `PosixPath.is_relative_to` once Python 3.8 is no # longer supported. filter_parts = path.parts From 5d26325f59880864863b5e56a08aa0f83b623f2d Mon Sep 17 00:00:00 2001 From: George Ogden Date: Thu, 27 Nov 2025 17:46:13 +0000 Subject: [PATCH 1232/1392] Allow Pathlike paths when creating a git repo --- git/repo/base.py | 2 +- test/test_repo.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/git/repo/base.py b/git/repo/base.py index fbed6e471..b1b95ce42 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -223,7 +223,7 @@ def __init__( epath = epath or path or os.getcwd() if not isinstance(epath, str): - epath = str(epath) + epath = epath.__fspath__() if expand_vars and re.search(self.re_envvars, epath): warnings.warn( "The use of environment variables in paths is deprecated" diff --git a/test/test_repo.py b/test/test_repo.py index dc2cfe7b1..cd22430a7 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -3,6 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +from dataclasses import dataclass import gc import glob import io @@ -105,6 +106,18 @@ def test_repo_creation_pathlib(self, rw_repo): r_from_gitdir = Repo(pathlib.Path(rw_repo.git_dir)) self.assertEqual(r_from_gitdir.git_dir, rw_repo.git_dir) + @with_rw_repo("0.3.2.1") + def test_repo_creation_pathlike(self, rw_repo): + @dataclass + class PathLikeMock: + path: str + + def __fspath__(self) -> str: + return self.path + + r_from_gitdir = Repo(PathLikeMock(rw_repo.git_dir)) + self.assertEqual(r_from_gitdir.git_dir, rw_repo.git_dir) + def test_description(self): txt = "Test repository" self.rorepo.description = txt From 59c3c8065a402c4cd8a71625f51b6792fdc04863 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Thu, 27 Nov 2025 19:20:07 +0000 Subject: [PATCH 1233/1392] Fix missing path conversion --- git/repo/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/repo/base.py b/git/repo/base.py index b1b95ce42..be50300b5 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -219,7 +219,7 @@ def __init__( # Given how the tests are written, this seems more likely to catch Cygwin # git used from Windows than Windows git used from Cygwin. Therefore # changing to Cygwin-style paths is the relevant operation. - epath = cygpath(str(epath)) + epath = cygpath(epath if isinstance(epath, str) else epath.__fspath__()) epath = epath or path or os.getcwd() if not isinstance(epath, str): From 91d4cc5ea05df04c82fcfd3e35a6af2e903cc554 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 08:57:54 +0000 Subject: [PATCH 1234/1392] Use os.fspath instead of __fspath__ for reading paths --- git/config.py | 2 +- git/repo/base.py | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/git/config.py b/git/config.py index 200c81bb7..e3081401d 100644 --- a/git/config.py +++ b/git/config.py @@ -634,7 +634,7 @@ def read(self) -> None: # type: ignore[override] self._read(file_path, file_path.name) else: # Assume a path if it is not a file-object. - file_path = cast(PathLike, file_path) + file_path = os.fspath(file_path) try: with open(file_path, "rb") as fp: file_ok = True diff --git a/git/repo/base.py b/git/repo/base.py index be50300b5..2d87a06b7 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -219,11 +219,9 @@ def __init__( # Given how the tests are written, this seems more likely to catch Cygwin # git used from Windows than Windows git used from Cygwin. Therefore # changing to Cygwin-style paths is the relevant operation. - epath = cygpath(epath if isinstance(epath, str) else epath.__fspath__()) + epath = cygpath(os.fspath(epath)) - epath = epath or path or os.getcwd() - if not isinstance(epath, str): - epath = epath.__fspath__() + epath = os.fspath(epath) if expand_vars and re.search(self.re_envvars, epath): warnings.warn( "The use of environment variables in paths is deprecated" @@ -1361,11 +1359,8 @@ def _clone( ) -> "Repo": odbt = kwargs.pop("odbt", odb_default_type) - # When pathlib.Path or other class-based path is passed - if not isinstance(url, str): - url = url.__fspath__() - if not isinstance(path, str): - path = path.__fspath__() + url = os.fspath(url) + path = os.fspath(path) ## A bug win cygwin's Git, when `--bare` or `--separate-git-dir` # it prepends the cwd or(?) the `url` into the `path, so:: From 497ca401fe094fcae11410a46518e8f56d7bd665 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 09:04:19 +0000 Subject: [PATCH 1235/1392] Pin mypy==1.18.2 --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 75e9e81fa..460597539 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,7 @@ coverage[toml] ddt >= 1.1.1, != 1.4.3 mock ; python_version < "3.8" -mypy +mypy==1.18.2 # pin mypy to avoid new errors pre-commit pytest >= 7.3.1 pytest-cov From 50762f112fef28230deea55c2d0ca344c6c6cb2c Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 09:08:42 +0000 Subject: [PATCH 1236/1392] Fail remote pipeline when mypy fails --- .github/workflows/pythonpackage.yml | 1 - pyproject.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 975c2e29d..4666f3480 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -103,7 +103,6 @@ jobs: PYTHON_VERSION: ${{ matrix.python-version }} # With new versions of mypy new issues might arise. This is a problem if there is # nobody able to fix them, so we have to ignore errors until that changes. - continue-on-error: true - name: Test with pytest run: | diff --git a/pyproject.toml b/pyproject.toml index 58ed81f17..149f2dc92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ testpaths = "test" # Space separated list of paths from root e.g test tests doc # filterwarnings ignore::WarningType # ignores those warnings [tool.mypy] -python_version = "3.8" files = ["git/", "test/deprecation/"] disallow_untyped_defs = true no_implicit_optional = true From 8469a1292f51d5e211e69849844f418d773268e1 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 09:37:54 +0000 Subject: [PATCH 1237/1392] Fix or ignore all mypy errors --- git/config.py | 6 ++---- git/diff.py | 6 ++++-- git/index/typ.py | 4 ++-- git/objects/commit.py | 2 +- git/objects/submodule/base.py | 3 ++- git/refs/head.py | 6 +----- git/refs/log.py | 2 +- git/refs/symbolic.py | 3 +-- git/refs/tag.py | 8 ++++---- git/repo/base.py | 10 +++------- git/repo/fun.py | 2 +- git/types.py | 6 +++--- git/util.py | 8 +++++--- test/deprecation/test_types.py | 2 +- test/lib/helper.py | 4 ++-- test/test_submodule.py | 2 +- 16 files changed, 34 insertions(+), 40 deletions(-) diff --git a/git/config.py b/git/config.py index 200c81bb7..458151d05 100644 --- a/git/config.py +++ b/git/config.py @@ -574,7 +574,7 @@ def _included_paths(self) -> List[Tuple[str, str]]: if keyword.endswith("/i"): value = re.sub( r"[a-zA-Z]", - lambda m: "[{}{}]".format(m.group().lower(), m.group().upper()), + lambda m: f"[{m.group().lower()!r}{m.group().upper()!r}]", value, ) if self._repo.git_dir: @@ -633,8 +633,6 @@ def read(self) -> None: # type: ignore[override] file_path = cast(IO[bytes], file_path) self._read(file_path, file_path.name) else: - # Assume a path if it is not a file-object. - file_path = cast(PathLike, file_path) try: with open(file_path, "rb") as fp: file_ok = True @@ -768,7 +766,7 @@ def _assure_writable(self, method_name: str) -> None: if self.read_only: raise IOError("Cannot execute non-constant method %s.%s" % (self, method_name)) - def add_section(self, section: str) -> None: + def add_section(self, section: str | cp._UNNAMED_SECTION) -> None: """Assures added options will stay in order.""" return super().add_section(section) diff --git a/git/diff.py b/git/diff.py index 9c6ae59e0..2b1fd928c 100644 --- a/git/diff.py +++ b/git/diff.py @@ -21,15 +21,17 @@ Any, Iterator, List, + Literal, Match, Optional, + Sequence, Tuple, TYPE_CHECKING, TypeVar, Union, cast, ) -from git.types import Literal, PathLike +from git.types import PathLike if TYPE_CHECKING: from subprocess import Popen @@ -289,7 +291,7 @@ class DiffIndex(List[T_Diff]): The class improves the diff handling convenience. """ - change_type = ("A", "C", "D", "R", "M", "T") + change_type: Sequence[Literal["A", "C", "D", "R", "M", "T"]] = ("A", "C", "D", "R", "M", "T") """Change type invariant identifying possible ways a blob can have changed: * ``A`` = Added diff --git a/git/index/typ.py b/git/index/typ.py index 4bcb604ab..927633a9f 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -192,7 +192,7 @@ def from_base(cls, base: "BaseIndexEntry") -> "IndexEntry": Instance of type :class:`BaseIndexEntry`. """ time = pack(">LL", 0, 0) - return IndexEntry((base.mode, base.binsha, base.flags, base.path, time, time, 0, 0, 0, 0, 0)) + return IndexEntry((base.mode, base.binsha, base.flags, base.path, time, time, 0, 0, 0, 0, 0)) # type: ignore[arg-type] @classmethod def from_blob(cls, blob: Blob, stage: int = 0) -> "IndexEntry": @@ -211,5 +211,5 @@ def from_blob(cls, blob: Blob, stage: int = 0) -> "IndexEntry": 0, 0, blob.size, - ) + ) # type: ignore[arg-type] ) diff --git a/git/objects/commit.py b/git/objects/commit.py index fbe0ee9c0..8c51254a2 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -900,7 +900,7 @@ def co_authors(self) -> List[Actor]: if self.message: results = re.findall( r"^Co-authored-by: (.*) <(.*?)>$", - self.message, + str(self.message), re.MULTILINE, ) for author in results: diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 5031a2e71..b4a4ca467 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -25,6 +25,7 @@ ) from git.objects.base import IndexObject, Object from git.objects.util import TraversableIterableObj +from ...refs.remote import RemoteReference from git.util import ( IterableList, RemoteProgress, @@ -355,7 +356,7 @@ def _clone_repo( module_checkout_path = osp.join(str(repo.working_tree_dir), path) if url.startswith("../"): - remote_name = repo.active_branch.tracking_branch().remote_name + remote_name = cast(RemoteReference, repo.active_branch.tracking_branch()).remote_name repo_remote_url = repo.remote(remote_name).url url = os.path.join(repo_remote_url, url) diff --git a/git/refs/head.py b/git/refs/head.py index 683634451..3c43993e7 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -22,7 +22,6 @@ from git.types import Commit_ish, PathLike if TYPE_CHECKING: - from git.objects import Commit from git.refs import RemoteReference from git.repo import Repo @@ -44,9 +43,6 @@ class HEAD(SymbolicReference): __slots__ = () - # TODO: This can be removed once SymbolicReference.commit has static type hints. - commit: "Commit" - def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME) -> None: if path != self._HEAD_NAME: raise ValueError("HEAD instance must point to %r, got %r" % (self._HEAD_NAME, path)) @@ -149,7 +145,7 @@ class Head(Reference): k_config_remote_ref = "merge" # Branch to merge from remote. @classmethod - def delete(cls, repo: "Repo", *heads: "Union[Head, str]", force: bool = False, **kwargs: Any) -> None: + def delete(cls, repo: "Repo", *heads: "Union[Head, str]", force: bool = False, **kwargs: Any) -> None: # type: ignore[override] """Delete the given heads. :param force: diff --git a/git/refs/log.py b/git/refs/log.py index 8f2f2cd38..4e3666993 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -145,7 +145,7 @@ def from_line(cls, line: bytes) -> "RefLogEntry": actor = Actor._from_string(info[82 : email_end + 1]) time, tz_offset = parse_date(info[email_end + 2 :]) # skipcq: PYL-W0621 - return RefLogEntry((oldhexsha, newhexsha, actor, (time, tz_offset), msg)) + return RefLogEntry(oldhexsha, newhexsha, actor, (time, tz_offset), msg) # type: ignore[call-arg] class RefLog(List[RefLogEntry], Serializable): diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 74bb1fe0a..f0d2abcf4 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -916,8 +916,7 @@ def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_Refere SymbolicReference, ): try: - instance: T_References - instance = ref_type(repo, path) + instance = cast(T_References, ref_type(repo, path)) if instance.__class__ is SymbolicReference and instance.is_detached: raise ValueError("SymbolicRef was detached, we drop it") else: diff --git a/git/refs/tag.py b/git/refs/tag.py index 1e38663ae..4525b09cb 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -45,8 +45,8 @@ class TagReference(Reference): _common_default = "tags" _common_path_default = Reference._common_path_default + "/" + _common_default - @property - def commit(self) -> "Commit": # type: ignore[override] # LazyMixin has unrelated commit method + @property # type: ignore[misc] + def commit(self) -> "Commit": # LazyMixin has unrelated commit method """:return: Commit object the tag ref points to :raise ValueError: @@ -80,8 +80,8 @@ def tag(self) -> Union["TagObject", None]: return None # Make object read-only. It should be reasonably hard to adjust an existing tag. - @property - def object(self) -> AnyGitObject: # type: ignore[override] + @property # type: ignore[misc] + def object(self) -> AnyGitObject: return Reference._get_object(self) @classmethod diff --git a/git/repo/base.py b/git/repo/base.py index 6ea96aad2..1ef7114af 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -684,11 +684,7 @@ def _config_reader( git_dir: Optional[PathLike] = None, ) -> GitConfigParser: if config_level is None: - files = [ - self._get_config_path(cast(Lit_config_levels, f), git_dir) - for f in self.config_level - if cast(Lit_config_levels, f) - ] + files = [self._get_config_path(f, git_dir) for f in self.config_level if f] else: files = [self._get_config_path(config_level, git_dir)] return GitConfigParser(files, read_only=True, repo=self) @@ -1484,7 +1480,7 @@ def clone( self.common_dir, path, type(self.odb), - progress, + progress, # type: ignore[arg-type] multi_options, allow_unsafe_protocols=allow_unsafe_protocols, allow_unsafe_options=allow_unsafe_options, @@ -1545,7 +1541,7 @@ def clone_from( url, to_path, GitCmdObjectDB, - progress, + progress, # type: ignore[arg-type] multi_options, allow_unsafe_protocols=allow_unsafe_protocols, allow_unsafe_options=allow_unsafe_options, diff --git a/git/repo/fun.py b/git/repo/fun.py index 1c995c6c6..3f00e60ea 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -286,7 +286,7 @@ def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: # END handle refname else: if ref is not None: - obj = cast("Commit", ref.commit) + obj = ref.commit # END handle ref # END initialize obj on first token diff --git a/git/types.py b/git/types.py index cce184530..c6dbb717b 100644 --- a/git/types.py +++ b/git/types.py @@ -13,7 +13,7 @@ Sequence as Sequence, Tuple, TYPE_CHECKING, - Type, + TypeAlias, TypeVar, Union, ) @@ -117,7 +117,7 @@ object types. """ -GitObjectTypeString = Literal["commit", "tag", "blob", "tree"] +GitObjectTypeString: TypeAlias = Literal["commit", "tag", "blob", "tree"] """Literal strings identifying git object types and the :class:`~git.objects.base.Object`-based types that represent them. @@ -130,7 +130,7 @@ https://git-scm.com/docs/gitglossary#def_object_type """ -Lit_commit_ish: Type[Literal["commit", "tag"]] +Lit_commit_ish: TypeAlias = Literal["commit", "tag"] """Deprecated. Type of literal strings identifying typically-commitish git object types. Prior to a bugfix, this type had been defined more broadly. Any usage is in practice diff --git a/git/util.py b/git/util.py index 0aff5eb64..54a5b7bd1 100644 --- a/git/util.py +++ b/git/util.py @@ -1143,7 +1143,7 @@ def _obtain_lock(self) -> None: # END endless loop -class IterableList(List[T_IterableObj]): +class IterableList(List[T_IterableObj]): # type: ignore[type-var] """List of iterable objects allowing to query an object by id or by named index:: heads = repo.heads @@ -1214,14 +1214,16 @@ def __getitem__(self, index: Union[SupportsIndex, int, slice, str]) -> T_Iterabl raise ValueError("Index should be an int or str") else: try: + if not isinstance(index, str): + raise AttributeError(f"{index} is not a valid attribute") return getattr(self, index) except AttributeError as e: - raise IndexError("No item found with id %r" % (self._prefix + index)) from e + raise IndexError(f"No item found with id {self._prefix}{index}") from e # END handle getattr def __delitem__(self, index: Union[SupportsIndex, int, slice, str]) -> None: delindex = cast(int, index) - if not isinstance(index, int): + if isinstance(index, str): delindex = -1 name = self._prefix + index for i, item in enumerate(self): diff --git a/test/deprecation/test_types.py b/test/deprecation/test_types.py index f97375a85..d3c6af645 100644 --- a/test/deprecation/test_types.py +++ b/test/deprecation/test_types.py @@ -36,7 +36,7 @@ def test_can_access_lit_commit_ish_but_it_is_not_usable() -> None: assert 'Literal["commit", "tag"]' in message, "Has new definition." assert "GitObjectTypeString" in message, "Has new type name for old definition." - _: Lit_commit_ish = "commit" # type: ignore[valid-type] + _: Lit_commit_ish = "commit" # It should be as documented (even though deliberately unusable in static checks). assert Lit_commit_ish == Literal["commit", "tag"] diff --git a/test/lib/helper.py b/test/lib/helper.py index 241d27341..b4615f400 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -149,7 +149,7 @@ def repo_creator(self): os.chdir(rw_repo.working_dir) try: return func(self, rw_repo) - except: # noqa: E722 B001 + except: # noqa: E722 _logger.info("Keeping repo after failure: %s", repo_dir) repo_dir = None raise @@ -309,7 +309,7 @@ def remote_repo_creator(self): with cwd(rw_repo.working_dir): try: return func(self, rw_repo, rw_daemon_repo) - except: # noqa: E722 B001 + except: # noqa: E722 _logger.info( "Keeping repos after failure: \n rw_repo_dir: %s \n rw_daemon_repo_dir: %s", rw_repo_dir, diff --git a/test/test_submodule.py b/test/test_submodule.py index 4a248eb60..a92dd8fd4 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -932,7 +932,7 @@ def assert_exists(sm, value=True): csm.repo.index.commit("Have to commit submodule change for algorithm to pick it up") assert csm.url == "bar" - self.assertRaises( + self.assertRaises( # noqa: B017 Exception, rsm.update, recursive=True, From a9756bc0c8997482a7f69cc8e46a9f461afea8f6 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 09:46:47 +0000 Subject: [PATCH 1238/1392] Fix typing so that code can run --- git/config.py | 2 +- git/objects/submodule/base.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/git/config.py b/git/config.py index 458151d05..bccf61258 100644 --- a/git/config.py +++ b/git/config.py @@ -766,7 +766,7 @@ def _assure_writable(self, method_name: str) -> None: if self.read_only: raise IOError("Cannot execute non-constant method %s.%s" % (self, method_name)) - def add_section(self, section: str | cp._UNNAMED_SECTION) -> None: + def add_section(self, section: "cp._SectionName") -> None: """Assures added options will stay in order.""" return super().add_section(section) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index b4a4ca467..20f3e9ccf 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -25,7 +25,6 @@ ) from git.objects.base import IndexObject, Object from git.objects.util import TraversableIterableObj -from ...refs.remote import RemoteReference from git.util import ( IterableList, RemoteProgress, @@ -67,7 +66,7 @@ if TYPE_CHECKING: from git.index import IndexFile from git.objects.commit import Commit - from git.refs import Head + from git.refs import Head, RemoteReference from git.repo import Repo # ----------------------------------------------------------------------------- @@ -356,7 +355,7 @@ def _clone_repo( module_checkout_path = osp.join(str(repo.working_tree_dir), path) if url.startswith("../"): - remote_name = cast(RemoteReference, repo.active_branch.tracking_branch()).remote_name + remote_name = cast("RemoteReference", repo.active_branch.tracking_branch()).remote_name repo_remote_url = repo.remote(remote_name).url url = os.path.join(repo_remote_url, url) From 0aba3e7bdec7544a86b6e6ba4b0ad8e2ac5cd2c7 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 10:02:07 +0000 Subject: [PATCH 1239/1392] Stop Lit_commit_ish being imported --- git/types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git/types.py b/git/types.py index c6dbb717b..100fff43f 100644 --- a/git/types.py +++ b/git/types.py @@ -13,7 +13,6 @@ Sequence as Sequence, Tuple, TYPE_CHECKING, - TypeAlias, TypeVar, Union, ) @@ -117,7 +116,7 @@ object types. """ -GitObjectTypeString: TypeAlias = Literal["commit", "tag", "blob", "tree"] +GitObjectTypeString = Literal["commit", "tag", "blob", "tree"] """Literal strings identifying git object types and the :class:`~git.objects.base.Object`-based types that represent them. @@ -130,7 +129,8 @@ https://git-scm.com/docs/gitglossary#def_object_type """ -Lit_commit_ish: TypeAlias = Literal["commit", "tag"] +if TYPE_CHECKING: + Lit_commit_ish = Literal["commit", "tag"] """Deprecated. Type of literal strings identifying typically-commitish git object types. Prior to a bugfix, this type had been defined more broadly. Any usage is in practice From 019f270785fd01558b48d21fe1469b9a2132d04b Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 10:02:45 +0000 Subject: [PATCH 1240/1392] Set __test__ = False in not tested classes --- test/test_remote.py | 5 ++++- test/test_submodule.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test_remote.py b/test/test_remote.py index 5ddb41bc0..b1d686f05 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -44,7 +44,7 @@ class TestRemoteProgress(RemoteProgress): __slots__ = ("_seen_lines", "_stages_per_op", "_num_progress_messages") - def __init__(self): + def __init__(self) -> None: super().__init__() self._seen_lines = [] self._stages_per_op = {} @@ -103,6 +103,9 @@ def assert_received_message(self): assert self._num_progress_messages +TestRemoteProgress.__test__ = False # type: ignore + + class TestRemote(TestBase): def tearDown(self): gc.collect() diff --git a/test/test_submodule.py b/test/test_submodule.py index a92dd8fd4..edff064c4 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -58,6 +58,7 @@ def update(self, op, cur_count, max_count, message=""): print(op, cur_count, max_count, message) +TestRootProgress.__test__ = False prog = TestRootProgress() From ca5a2e817829861c5a0830806c0a40a33a5ab83f Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 11:40:23 +0000 Subject: [PATCH 1241/1392] Add missing parentheses around tuple constructor --- git/refs/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/refs/log.py b/git/refs/log.py index 4e3666993..4751cff99 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -145,7 +145,7 @@ def from_line(cls, line: bytes) -> "RefLogEntry": actor = Actor._from_string(info[82 : email_end + 1]) time, tz_offset = parse_date(info[email_end + 2 :]) # skipcq: PYL-W0621 - return RefLogEntry(oldhexsha, newhexsha, actor, (time, tz_offset), msg) # type: ignore[call-arg] + return RefLogEntry((oldhexsha, newhexsha, actor, (time, tz_offset), msg)) # type: ignore [arg-type] class RefLog(List[RefLogEntry], Serializable): From c75790837d0fd5bb9a7ba26a48b957b5d70987fb Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 11:43:21 +0000 Subject: [PATCH 1242/1392] Install mypy for Python >= 3.9 --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 460597539..e6e01c683 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,7 @@ coverage[toml] ddt >= 1.1.1, != 1.4.3 mock ; python_version < "3.8" -mypy==1.18.2 # pin mypy to avoid new errors +mypy==1.18.2 ; python_version >= "3.9" # pin mypy version to avoid new errors pre-commit pytest >= 7.3.1 pytest-cov From 9decf740ad2f1d89b55bda3a42880fa4f7b652ea Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 11:49:10 +0000 Subject: [PATCH 1243/1392] Skip mypy when Python < 3.9 --- .github/workflows/pythonpackage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4666f3480..9e05b3fe6 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -95,6 +95,7 @@ jobs: continue-on-error: true - name: Check types with mypy + if: matrix.python-version != '3.7' && matrix.python-version != '3.8' run: | mypy --python-version="${PYTHON_VERSION%t}" # Version only, with no "t" for free-threaded. env: From a1f094c81fcf4a6b559c2a26fc622c89e4f19735 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 12:10:04 +0000 Subject: [PATCH 1244/1392] Use git.types.Literal instead of typing.Literal --- git/diff.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/git/diff.py b/git/diff.py index 2b1fd928c..23cb5675e 100644 --- a/git/diff.py +++ b/git/diff.py @@ -21,7 +21,6 @@ Any, Iterator, List, - Literal, Match, Optional, Sequence, @@ -31,7 +30,7 @@ Union, cast, ) -from git.types import PathLike +from git.types import PathLike, Literal if TYPE_CHECKING: from subprocess import Popen @@ -291,7 +290,7 @@ class DiffIndex(List[T_Diff]): The class improves the diff handling convenience. """ - change_type: Sequence[Literal["A", "C", "D", "R", "M", "T"]] = ("A", "C", "D", "R", "M", "T") + change_type: Sequence[Literal["A", "C", "D", "R", "M", "T"]] = ("A", "C", "D", "R", "M", "T") # noqa: F821 """Change type invariant identifying possible ways a blob can have changed: * ``A`` = Added From 0414bf78cfc7d1889121c414efa7841f57343984 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 21:28:17 +0000 Subject: [PATCH 1245/1392] Replace extra occurrences of str with fspath --- git/config.py | 2 +- git/index/base.py | 18 +++++++++--------- git/index/fun.py | 4 ++-- git/index/util.py | 2 +- git/objects/blob.py | 3 ++- git/objects/submodule/base.py | 10 +++++----- git/refs/reference.py | 3 ++- git/refs/symbolic.py | 18 +++++++++--------- git/repo/base.py | 4 ++-- git/util.py | 20 ++++++++++---------- test/lib/helper.py | 11 +++++++++++ test/test_clone.py | 11 +---------- test/test_index.py | 22 ++++++++++++---------- test/test_repo.py | 10 +--------- 14 files changed, 68 insertions(+), 70 deletions(-) diff --git a/git/config.py b/git/config.py index e3081401d..732f347f0 100644 --- a/git/config.py +++ b/git/config.py @@ -578,7 +578,7 @@ def _included_paths(self) -> List[Tuple[str, str]]: value, ) if self._repo.git_dir: - if fnmatch.fnmatchcase(str(self._repo.git_dir), value): + if fnmatch.fnmatchcase(os.fspath(self._repo.git_dir), value): paths += self.items(section) elif keyword == "onbranch": diff --git a/git/index/base.py b/git/index/base.py index 7cc9d3ade..905feb076 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -404,10 +404,10 @@ def _iter_expand_paths(self: "IndexFile", paths: Sequence[PathLike]) -> Iterator def raise_exc(e: Exception) -> NoReturn: raise e - r = str(self.repo.working_tree_dir) + r = os.fspath(self.repo.working_tree_dir) rs = r + os.sep for path in paths: - abs_path = str(path) + abs_path = os.fspath(path) if not osp.isabs(abs_path): abs_path = osp.join(r, path) # END make absolute path @@ -434,7 +434,7 @@ def raise_exc(e: Exception) -> NoReturn: # characters. if abs_path not in resolved_paths: for f in self._iter_expand_paths(glob.glob(abs_path)): - yield str(f).replace(rs, "") + yield os.fspath(f).replace(rs, "") continue # END glob handling try: @@ -569,7 +569,7 @@ def resolve_blobs(self, iter_blobs: Iterator[Blob]) -> "IndexFile": for blob in iter_blobs: stage_null_key = (blob.path, 0) if stage_null_key in self.entries: - raise ValueError("Path %r already exists at stage 0" % str(blob.path)) + raise ValueError("Path %r already exists at stage 0" % os.fspath(blob.path)) # END assert blob is not stage 0 already # Delete all possible stages. @@ -656,10 +656,10 @@ def _to_relative_path(self, path: PathLike) -> PathLike: return path if self.repo.bare: raise InvalidGitRepositoryError("require non-bare repository") - if not osp.normpath(str(path)).startswith(str(self.repo.working_tree_dir)): + if not osp.normpath(os.fspath(path)).startswith(os.fspath(self.repo.working_tree_dir)): raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir)) result = os.path.relpath(path, self.repo.working_tree_dir) - if str(path).endswith(os.sep) and not result.endswith(os.sep): + if os.fspath(path).endswith(os.sep) and not result.endswith(os.sep): result += os.sep return result @@ -731,7 +731,7 @@ def _entries_for_paths( for path in paths: if osp.isabs(path): abspath = path - gitrelative_path = path[len(str(self.repo.working_tree_dir)) + 1 :] + gitrelative_path = path[len(os.fspath(self.repo.working_tree_dir)) + 1 :] else: gitrelative_path = path if self.repo.working_tree_dir: @@ -1359,11 +1359,11 @@ def make_exc() -> GitCommandError: try: self.entries[(co_path, 0)] except KeyError: - folder = str(co_path) + folder = os.fspath(co_path) if not folder.endswith("/"): folder += "/" for entry in self.entries.values(): - if str(entry.path).startswith(folder): + if os.fspath(entry.path).startswith(folder): p = entry.path self._write_path_to_stdin(proc, p, p, make_exc, fprogress, read_from_stdout=False) checked_out_files.append(p) diff --git a/git/index/fun.py b/git/index/fun.py index 0b3d79cf1..845221c61 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -87,7 +87,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: return env = os.environ.copy() - env["GIT_INDEX_FILE"] = safe_decode(str(index.path)) + env["GIT_INDEX_FILE"] = safe_decode(os.fspath(index.path)) env["GIT_EDITOR"] = ":" cmd = [hp] try: @@ -167,7 +167,7 @@ def write_cache( beginoffset = tell() write(entry.ctime_bytes) # ctime write(entry.mtime_bytes) # mtime - path_str = str(entry.path) + path_str = os.fspath(entry.path) path: bytes = force_bytes(path_str, encoding=defenc) plen = len(path) & CE_NAMEMASK # Path length assert plen == len(path), "Path %s too long to fit into index" % entry.path diff --git a/git/index/util.py b/git/index/util.py index e59cb609f..2d2422ab4 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -106,7 +106,7 @@ def git_working_dir(func: Callable[..., _T]) -> Callable[..., _T]: @wraps(func) def set_git_working_dir(self: "IndexFile", *args: Any, **kwargs: Any) -> _T: cur_wd = os.getcwd() - os.chdir(str(self.repo.working_tree_dir)) + os.chdir(os.fspath(self.repo.working_tree_dir)) try: return func(self, *args, **kwargs) finally: diff --git a/git/objects/blob.py b/git/objects/blob.py index 58de59642..f7d49c9cc 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -6,6 +6,7 @@ __all__ = ["Blob"] from mimetypes import guess_type +import os import sys if sys.version_info >= (3, 8): @@ -44,5 +45,5 @@ def mime_type(self) -> str: """ guesses = None if self.path: - guesses = guess_type(str(self.path)) + guesses = guess_type(os.fspath(self.path)) return guesses and guesses[0] or self.DEFAULT_MIME_TYPE diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 5031a2e71..cc1f94c6c 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -352,7 +352,7 @@ def _clone_repo( module_abspath_dir = osp.dirname(module_abspath) if not osp.isdir(module_abspath_dir): os.makedirs(module_abspath_dir) - module_checkout_path = osp.join(str(repo.working_tree_dir), path) + module_checkout_path = osp.join(os.fspath(repo.working_tree_dir), path) if url.startswith("../"): remote_name = repo.active_branch.tracking_branch().remote_name @@ -541,7 +541,7 @@ def add( if sm.exists(): # Reretrieve submodule from tree. try: - sm = repo.head.commit.tree[str(path)] + sm = repo.head.commit.tree[os.fspath(path)] sm._name = name return sm except KeyError: @@ -1016,7 +1016,7 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = return self # END handle no change - module_checkout_abspath = join_path_native(str(self.repo.working_tree_dir), module_checkout_path) + module_checkout_abspath = join_path_native(os.fspath(self.repo.working_tree_dir), module_checkout_path) if osp.isfile(module_checkout_abspath): raise ValueError("Cannot move repository onto a file: %s" % module_checkout_abspath) # END handle target files @@ -1313,7 +1313,7 @@ def set_parent_commit(self, commit: Union[Commit_ish, str, None], check: bool = # If check is False, we might see a parent-commit that doesn't even contain the # submodule anymore. in that case, mark our sha as being NULL. try: - self.binsha = pctree[str(self.path)].binsha + self.binsha = pctree[os.fspath(self.path)].binsha except KeyError: self.binsha = self.NULL_BIN_SHA @@ -1395,7 +1395,7 @@ def rename(self, new_name: str) -> "Submodule": destination_module_abspath = self._module_abspath(self.repo, self.path, new_name) source_dir = mod.git_dir # Let's be sure the submodule name is not so obviously tied to a directory. - if str(destination_module_abspath).startswith(str(mod.git_dir)): + if os.fspath(destination_module_abspath).startswith(os.fspath(mod.git_dir)): tmp_dir = self._module_abspath(self.repo, self.path, str(uuid.uuid4())) os.renames(source_dir, tmp_dir) source_dir = tmp_dir diff --git a/git/refs/reference.py b/git/refs/reference.py index e5d473779..0c4327225 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -3,6 +3,7 @@ __all__ = ["Reference"] +import os from git.util import IterableObj, LazyMixin from .symbolic import SymbolicReference, T_References @@ -65,7 +66,7 @@ def __init__(self, repo: "Repo", path: PathLike, check_path: bool = True) -> Non If ``False``, you can provide any path. Otherwise the path must start with the default path prefix of this type. """ - if check_path and not str(path).startswith(self._common_path_default + "/"): + if check_path and not os.fspath(path).startswith(self._common_path_default + "/"): raise ValueError(f"Cannot instantiate {self.__class__.__name__!r} from path {path}") self.path: str # SymbolicReference converts to string at the moment. super().__init__(repo, path) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 74bb1fe0a..c7db129d9 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -79,7 +79,7 @@ def __init__(self, repo: "Repo", path: PathLike, check_path: bool = False) -> No self.path = path def __str__(self) -> str: - return str(self.path) + return os.fspath(self.path) def __repr__(self) -> str: return '' % (self.__class__.__name__, self.path) @@ -103,7 +103,7 @@ def name(self) -> str: In case of symbolic references, the shortest assumable name is the path itself. """ - return str(self.path) + return os.fspath(self.path) @property def abspath(self) -> PathLike: @@ -178,7 +178,7 @@ def _check_ref_name_valid(ref_path: PathLike) -> None: """ previous: Union[str, None] = None one_before_previous: Union[str, None] = None - for c in str(ref_path): + for c in os.fspath(ref_path): if c in " ~^:?*[\\": raise ValueError( f"Invalid reference '{ref_path}': references cannot contain spaces, tildes (~), carets (^)," @@ -212,7 +212,7 @@ def _check_ref_name_valid(ref_path: PathLike) -> None: raise ValueError(f"Invalid reference '{ref_path}': references cannot end with a forward slash (/)") elif previous == "@" and one_before_previous is None: raise ValueError(f"Invalid reference '{ref_path}': references cannot be '@'") - elif any(component.endswith(".lock") for component in str(ref_path).split("/")): + elif any(component.endswith(".lock") for component in os.fspath(ref_path).split("/")): raise ValueError( f"Invalid reference '{ref_path}': references cannot have slash-separated components that end with" " '.lock'" @@ -235,7 +235,7 @@ def _get_ref_info_helper( tokens: Union[None, List[str], Tuple[str, str]] = None repodir = _git_dir(repo, ref_path) try: - with open(os.path.join(repodir, str(ref_path)), "rt", encoding="UTF-8") as fp: + with open(os.path.join(repodir, os.fspath(ref_path)), "rt", encoding="UTF-8") as fp: value = fp.read().rstrip() # Don't only split on spaces, but on whitespace, which allows to parse lines like: # 60b64ef992065e2600bfef6187a97f92398a9144 branch 'master' of git-server:/path/to/repo @@ -614,7 +614,7 @@ def to_full_path(cls, path: Union[PathLike, "SymbolicReference"]) -> PathLike: full_ref_path = path if not cls._common_path_default: return full_ref_path - if not str(path).startswith(cls._common_path_default + "/"): + if not os.fspath(path).startswith(cls._common_path_default + "/"): full_ref_path = "%s/%s" % (cls._common_path_default, path) return full_ref_path @@ -706,7 +706,7 @@ def _create( if not force and os.path.isfile(abs_ref_path): target_data = str(target) if isinstance(target, SymbolicReference): - target_data = str(target.path) + target_data = os.fspath(target.path) if not resolve: target_data = "ref: " + target_data with open(abs_ref_path, "rb") as fd: @@ -842,7 +842,7 @@ def _iter_items( # Read packed refs. for _sha, rela_path in cls._iter_packed_refs(repo): - if rela_path.startswith(str(common_path)): + if rela_path.startswith(os.fspath(common_path)): rela_paths.add(rela_path) # END relative path matches common path # END packed refs reading @@ -931,4 +931,4 @@ def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_Refere def is_remote(self) -> bool: """:return: True if this symbolic reference points to a remote branch""" - return str(self.path).startswith(self._remote_common_path_default + "/") + return os.fspath(self.path).startswith(self._remote_common_path_default + "/") diff --git a/git/repo/base.py b/git/repo/base.py index 2d87a06b7..2fe18f48c 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -554,7 +554,7 @@ def tag(self, path: PathLike) -> TagReference: @staticmethod def _to_full_tag_path(path: PathLike) -> str: - path_str = str(path) + path_str = os.fspath(path) if path_str.startswith(TagReference._common_path_default + "/"): return path_str if path_str.startswith(TagReference._common_default + "/"): @@ -959,7 +959,7 @@ def is_dirty( if not submodules: default_args.append("--ignore-submodules") if path: - default_args.extend(["--", str(path)]) + default_args.extend(["--", os.fspath(path)]) if index: # diff index against HEAD. if osp.isfile(self.index.path) and len(self.git.diff("--cached", *default_args)): diff --git a/git/util.py b/git/util.py index 0aff5eb64..f18eb6e52 100644 --- a/git/util.py +++ b/git/util.py @@ -272,9 +272,9 @@ def stream_copy(source: BinaryIO, destination: BinaryIO, chunk_size: int = 512 * def join_path(a: PathLike, *p: PathLike) -> PathLike: R"""Join path tokens together similar to osp.join, but always use ``/`` instead of possibly ``\`` on Windows.""" - path = str(a) + path = os.fspath(a) for b in p: - b = str(b) + b = os.fspath(b) if not b: continue if b.startswith("/"): @@ -290,18 +290,18 @@ def join_path(a: PathLike, *p: PathLike) -> PathLike: if sys.platform == "win32": def to_native_path_windows(path: PathLike) -> PathLike: - path = str(path) + path = os.fspath(path) return path.replace("/", "\\") def to_native_path_linux(path: PathLike) -> str: - path = str(path) + path = os.fspath(path) return path.replace("\\", "/") to_native_path = to_native_path_windows else: # No need for any work on Linux. def to_native_path_linux(path: PathLike) -> str: - return str(path) + return os.fspath(path) to_native_path = to_native_path_linux @@ -372,7 +372,7 @@ def is_exec(fpath: str) -> bool: progs = [] if not path: path = os.environ["PATH"] - for folder in str(path).split(os.pathsep): + for folder in os.fspath(path).split(os.pathsep): folder = folder.strip('"') if folder: exe_path = osp.join(folder, program) @@ -397,7 +397,7 @@ def _cygexpath(drive: Optional[str], path: str) -> str: p = cygpath(p) elif drive: p = "/proc/cygdrive/%s/%s" % (drive.lower(), p) - p_str = str(p) # ensure it is a str and not AnyPath + p_str = os.fspath(p) # ensure it is a str and not AnyPath return p_str.replace("\\", "/") @@ -418,7 +418,7 @@ def _cygexpath(drive: Optional[str], path: str) -> str: def cygpath(path: str) -> str: """Use :meth:`git.cmd.Git.polish_url` instead, that works on any environment.""" - path = str(path) # Ensure is str and not AnyPath. + path = os.fspath(path) # Ensure is str and not AnyPath. # Fix to use Paths when 3.5 dropped. Or to be just str if only for URLs? if not path.startswith(("/cygdrive", "//", "/proc/cygdrive")): for regex, parser, recurse in _cygpath_parsers: @@ -438,7 +438,7 @@ def cygpath(path: str) -> str: def decygpath(path: PathLike) -> str: - path = str(path) + path = os.fspath(path) m = _decygpath_regex.match(path) if m: drive, rest_path = m.groups() @@ -497,7 +497,7 @@ def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: elif git_executable is None: return False else: - return _is_cygwin_git(str(git_executable)) + return _is_cygwin_git(os.fspath(git_executable)) def get_user_id() -> str: diff --git a/test/lib/helper.py b/test/lib/helper.py index 241d27341..e51f428e3 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -10,6 +10,7 @@ "with_rw_directory", "with_rw_repo", "with_rw_and_rw_remote_repo", + "PathLikeMock", "TestBase", "VirtualEnvironment", "TestCase", @@ -20,6 +21,7 @@ ] import contextlib +from dataclasses import dataclass from functools import wraps import gc import io @@ -49,6 +51,15 @@ _logger = logging.getLogger(__name__) + +@dataclass +class PathLikeMock: + path: str + + def __fspath__(self) -> str: + return self.path + + # { Routines diff --git a/test/test_clone.py b/test/test_clone.py index 489931458..143a3b51f 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -1,7 +1,6 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from dataclasses import dataclass import os import os.path as osp import pathlib @@ -12,7 +11,7 @@ from git import GitCommandError, Repo from git.exc import UnsafeOptionError, UnsafeProtocolError -from test.lib import TestBase, with_rw_directory, with_rw_repo +from test.lib import TestBase, with_rw_directory, with_rw_repo, PathLikeMock from pathlib import Path import re @@ -51,14 +50,6 @@ def test_clone_from_pathlib(self, rw_dir): @with_rw_directory def test_clone_from_pathlike(self, rw_dir): original_repo = Repo.init(osp.join(rw_dir, "repo")) - - @dataclass - class PathLikeMock: - path: str - - def __fspath__(self) -> str: - return self.path - Repo.clone_from(PathLikeMock(original_repo.git_dir), PathLikeMock(os.path.join(rw_dir, "clone_pathlike"))) @with_rw_directory diff --git a/test/test_index.py b/test/test_index.py index 711b43a0b..c1db4166b 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -37,14 +37,7 @@ from git.objects import Blob from git.util import Actor, cwd, hex_to_bin, rmtree -from test.lib import ( - TestBase, - VirtualEnvironment, - fixture, - fixture_path, - with_rw_directory, - with_rw_repo, -) +from test.lib import TestBase, VirtualEnvironment, fixture, fixture_path, with_rw_directory, with_rw_repo, PathLikeMock HOOKS_SHEBANG = "#!/usr/bin/env sh\n" @@ -586,7 +579,7 @@ def mixed_iterator(): if type_id == 0: # path (str) yield entry.path elif type_id == 1: # path (PathLike) - yield Path(entry.path) + yield PathLikeMock(entry.path) elif type_id == 2: # blob yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) elif type_id == 3: # BaseIndexEntry @@ -1198,7 +1191,7 @@ def test_commit_msg_hook_fail(self, rw_repo): raise AssertionError("Should have caught a HookExecutionError") @with_rw_repo("HEAD") - def test_index_add_pathlike(self, rw_repo): + def test_index_add_pathlib(self, rw_repo): git_dir = Path(rw_repo.git_dir) file = git_dir / "file.txt" @@ -1206,6 +1199,15 @@ def test_index_add_pathlike(self, rw_repo): rw_repo.index.add(file) + @with_rw_repo("HEAD") + def test_index_add_pathlike(self, rw_repo): + git_dir = Path(rw_repo.git_dir) + + file = git_dir / "file.txt" + file.touch() + + rw_repo.index.add(PathLikeMock(str(file))) + @with_rw_repo("HEAD") def test_index_add_non_normalized_path(self, rw_repo): git_dir = Path(rw_repo.git_dir) diff --git a/test/test_repo.py b/test/test_repo.py index cd22430a7..f089bf6b8 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -3,7 +3,6 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -from dataclasses import dataclass import gc import glob import io @@ -41,7 +40,7 @@ from git.repo.fun import touch from git.util import bin_to_hex, cwd, cygpath, join_path_native, rmfile, rmtree -from test.lib import TestBase, fixture, with_rw_directory, with_rw_repo +from test.lib import TestBase, fixture, with_rw_directory, with_rw_repo, PathLikeMock def iter_flatten(lol): @@ -108,13 +107,6 @@ def test_repo_creation_pathlib(self, rw_repo): @with_rw_repo("0.3.2.1") def test_repo_creation_pathlike(self, rw_repo): - @dataclass - class PathLikeMock: - path: str - - def __fspath__(self) -> str: - return self.path - r_from_gitdir = Repo(PathLikeMock(rw_repo.git_dir)) self.assertEqual(r_from_gitdir.git_dir, rw_repo.git_dir) From 3801505d1218242e853dda17e981e2a2fa795b0e Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 21:43:09 +0000 Subject: [PATCH 1246/1392] Convert paths in constructors and large function calls --- git/index/base.py | 5 +++-- git/index/util.py | 2 +- git/refs/head.py | 2 ++ git/refs/log.py | 3 ++- git/refs/symbolic.py | 2 +- git/repo/fun.py | 1 + git/util.py | 2 +- 7 files changed, 11 insertions(+), 6 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 905feb076..cf7220ef8 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -652,14 +652,15 @@ def _to_relative_path(self, path: PathLike) -> PathLike: :raise ValueError: """ + path = os.fspath(path) if not osp.isabs(path): return path if self.repo.bare: raise InvalidGitRepositoryError("require non-bare repository") - if not osp.normpath(os.fspath(path)).startswith(os.fspath(self.repo.working_tree_dir)): + if not osp.normpath(path).startswith(os.fspath(self.repo.working_tree_dir)): raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir)) result = os.path.relpath(path, self.repo.working_tree_dir) - if os.fspath(path).endswith(os.sep) and not result.endswith(os.sep): + if path.endswith(os.sep) and not result.endswith(os.sep): result += os.sep return result diff --git a/git/index/util.py b/git/index/util.py index 2d2422ab4..231634cd6 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -37,7 +37,7 @@ class TemporaryFileSwap: __slots__ = ("file_path", "tmp_file_path") def __init__(self, file_path: PathLike) -> None: - self.file_path = file_path + self.file_path = os.fspath(file_path) dirname, basename = osp.split(file_path) fd, self.tmp_file_path = tempfile.mkstemp(prefix=basename, dir=dirname) os.close(fd) diff --git a/git/refs/head.py b/git/refs/head.py index 683634451..03daa3973 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -8,6 +8,7 @@ __all__ = ["HEAD", "Head"] +import os from git.config import GitConfigParser, SectionConstraint from git.exc import GitCommandError from git.util import join_path @@ -48,6 +49,7 @@ class HEAD(SymbolicReference): commit: "Commit" def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME) -> None: + path = os.fspath(path) if path != self._HEAD_NAME: raise ValueError("HEAD instance must point to %r, got %r" % (self._HEAD_NAME, path)) super().__init__(repo, path) diff --git a/git/refs/log.py b/git/refs/log.py index 8f2f2cd38..48bb02c60 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -4,6 +4,7 @@ __all__ = ["RefLog", "RefLogEntry"] from mmap import mmap +import os import os.path as osp import re import time as _time @@ -167,7 +168,7 @@ def __init__(self, filepath: Union[PathLike, None] = None) -> None: """Initialize this instance with an optional filepath, from which we will initialize our data. The path is also used to write changes back using the :meth:`write` method.""" - self._path = filepath + self._path = os.fspath(filepath) if filepath is not None: self._read_from_file() # END handle filepath diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index c7db129d9..24a72257d 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -76,7 +76,7 @@ class SymbolicReference: def __init__(self, repo: "Repo", path: PathLike, check_path: bool = False) -> None: self.repo = repo - self.path = path + self.path = os.fspath(path) def __str__(self) -> str: return os.fspath(self.path) diff --git a/git/repo/fun.py b/git/repo/fun.py index 1c995c6c6..1c7cfcb04 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -62,6 +62,7 @@ def is_git_dir(d: PathLike) -> bool: clearly indicates that we don't support it. There is the unlikely danger to throw if we see directories which just look like a worktree dir, but are none. """ + d = os.fspath(d) if osp.isdir(d): if (osp.isdir(osp.join(d, "objects")) or "GIT_OBJECT_DIRECTORY" in os.environ) and osp.isdir( osp.join(d, "refs") diff --git a/git/util.py b/git/util.py index f18eb6e52..5326af9d1 100644 --- a/git/util.py +++ b/git/util.py @@ -1011,7 +1011,7 @@ class LockFile: __slots__ = ("_file_path", "_owns_lock") def __init__(self, file_path: PathLike) -> None: - self._file_path = file_path + self._file_path = os.fspath(file_path) self._owns_lock = False def __del__(self) -> None: From 086e83239388988772e21ee820c23e59533382f7 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 28 Nov 2025 21:48:38 +0000 Subject: [PATCH 1247/1392] Fix union type conversion to path --- git/refs/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/refs/log.py b/git/refs/log.py index 48bb02c60..d1cc393f4 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -168,7 +168,7 @@ def __init__(self, filepath: Union[PathLike, None] = None) -> None: """Initialize this instance with an optional filepath, from which we will initialize our data. The path is also used to write changes back using the :meth:`write` method.""" - self._path = os.fspath(filepath) + self._path = None if filepath is None else os.fspath(filepath) if filepath is not None: self._read_from_file() # END handle filepath From b5c834af59531456551d406eb857934e7e87f1ce Mon Sep 17 00:00:00 2001 From: George Ogden Date: Sat, 29 Nov 2025 11:49:23 +0000 Subject: [PATCH 1248/1392] Remve comment about skipping mypy --- .github/workflows/pythonpackage.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 9e05b3fe6..ac764d9a7 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -102,8 +102,6 @@ jobs: MYPY_FORCE_COLOR: "1" TERM: "xterm-256color" # For color: https://github.com/python/mypy/issues/13817 PYTHON_VERSION: ${{ matrix.python-version }} - # With new versions of mypy new issues might arise. This is a problem if there is - # nobody able to fix them, so we have to ignore errors until that changes. - name: Test with pytest run: | From eb15123b82dbd13f9cc88606b8a580c424335fe7 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Sat, 29 Nov 2025 11:51:47 +0000 Subject: [PATCH 1249/1392] Use cast to allow silent getattrs --- git/util.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/git/util.py b/git/util.py index 54a5b7bd1..1f1595f1c 100644 --- a/git/util.py +++ b/git/util.py @@ -1214,9 +1214,7 @@ def __getitem__(self, index: Union[SupportsIndex, int, slice, str]) -> T_Iterabl raise ValueError("Index should be an int or str") else: try: - if not isinstance(index, str): - raise AttributeError(f"{index} is not a valid attribute") - return getattr(self, index) + return getattr(self, cast(str, index)) except AttributeError as e: raise IndexError(f"No item found with id {self._prefix}{index}") from e # END handle getattr From b3908ed04815b1d89a000fc7824a804c37906365 Mon Sep 17 00:00:00 2001 From: George Ogden <38294960+George-Ogden@users.noreply.github.com> Date: Sat, 29 Nov 2025 11:56:57 +0000 Subject: [PATCH 1250/1392] Use converted file path Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- git/index/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/index/util.py b/git/index/util.py index 231634cd6..80f0b014c 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -38,7 +38,7 @@ class TemporaryFileSwap: def __init__(self, file_path: PathLike) -> None: self.file_path = os.fspath(file_path) - dirname, basename = osp.split(file_path) + dirname, basename = osp.split(self.file_path) fd, self.tmp_file_path = tempfile.mkstemp(prefix=basename, dir=dirname) os.close(fd) with contextlib.suppress(OSError): # It may be that the source does not exist. From 50aea998641248f501735421ddc6165cbdb5d08c Mon Sep 17 00:00:00 2001 From: George Ogden <38294960+George-Ogden@users.noreply.github.com> Date: Sat, 29 Nov 2025 11:57:25 +0000 Subject: [PATCH 1251/1392] Remove redundant `fspath` Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- git/refs/symbolic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 24a72257d..7cf812416 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -79,7 +79,7 @@ def __init__(self, repo: "Repo", path: PathLike, check_path: bool = False) -> No self.path = os.fspath(path) def __str__(self) -> str: - return os.fspath(self.path) + return self.path def __repr__(self) -> str: return '' % (self.__class__.__name__, self.path) From 57a3af1ddc9b03f59a3e6d3f012c6043905763c0 Mon Sep 17 00:00:00 2001 From: George Ogden <38294960+George-Ogden@users.noreply.github.com> Date: Sat, 29 Nov 2025 11:57:55 +0000 Subject: [PATCH 1252/1392] Remove redundant `fspath` Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- git/refs/symbolic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 7cf812416..3cadb5061 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -103,7 +103,7 @@ def name(self) -> str: In case of symbolic references, the shortest assumable name is the path itself. """ - return os.fspath(self.path) + return self.path @property def abspath(self) -> PathLike: From df8087a2c90e25692eb8d4f09c8726fc78e21d05 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Sat, 29 Nov 2025 12:24:01 +0000 Subject: [PATCH 1253/1392] Remove a large number of redundant fspaths --- git/index/base.py | 2 +- git/index/fun.py | 2 +- git/objects/submodule/base.py | 2 +- git/refs/symbolic.py | 4 ++-- git/repo/base.py | 6 +++--- git/util.py | 3 +-- test/test_index.py | 8 +++++--- 7 files changed, 14 insertions(+), 13 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index cf7220ef8..2489949c1 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1360,7 +1360,7 @@ def make_exc() -> GitCommandError: try: self.entries[(co_path, 0)] except KeyError: - folder = os.fspath(co_path) + folder = co_path if not folder.endswith("/"): folder += "/" for entry in self.entries.values(): diff --git a/git/index/fun.py b/git/index/fun.py index 845221c61..9832aea6b 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -87,7 +87,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: return env = os.environ.copy() - env["GIT_INDEX_FILE"] = safe_decode(os.fspath(index.path)) + env["GIT_INDEX_FILE"] = safe_decode(index.path) env["GIT_EDITOR"] = ":" cmd = [hp] try: diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index cc1f94c6c..ffc1d3595 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1229,7 +1229,7 @@ def remove( wtd = mod.working_tree_dir del mod # Release file-handles (Windows). gc.collect() - rmtree(str(wtd)) + rmtree(wtd) # END delete tree if possible # END handle force diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 3cadb5061..557e8f5b4 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -706,7 +706,7 @@ def _create( if not force and os.path.isfile(abs_ref_path): target_data = str(target) if isinstance(target, SymbolicReference): - target_data = os.fspath(target.path) + target_data = target.path if not resolve: target_data = "ref: " + target_data with open(abs_ref_path, "rb") as fd: @@ -931,4 +931,4 @@ def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_Refere def is_remote(self) -> bool: """:return: True if this symbolic reference points to a remote branch""" - return os.fspath(self.path).startswith(self._remote_common_path_default + "/") + return self.path.startswith(self._remote_common_path_default + "/") diff --git a/git/repo/base.py b/git/repo/base.py index 2fe18f48c..e721aea40 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -126,7 +126,7 @@ class Repo: working_dir: PathLike """The working directory of the git command.""" - _working_tree_dir: Optional[PathLike] = None + _working_tree_dir: Optional[str] = None git_dir: PathLike """The ``.git`` repository directory.""" @@ -306,7 +306,7 @@ def __init__( self._working_tree_dir = None # END working dir handling - self.working_dir: PathLike = self._working_tree_dir or self.common_dir + self.working_dir: str = self._working_tree_dir or self.common_dir self.git = self.GitCommandWrapperType(self.working_dir) # Special handling, in special times. @@ -366,7 +366,7 @@ def description(self, descr: str) -> None: fp.write((descr + "\n").encode(defenc)) @property - def working_tree_dir(self) -> Optional[PathLike]: + def working_tree_dir(self) -> Optional[str]: """ :return: The working tree directory of our git repository. diff --git a/git/util.py b/git/util.py index 5326af9d1..94452ab17 100644 --- a/git/util.py +++ b/git/util.py @@ -397,8 +397,7 @@ def _cygexpath(drive: Optional[str], path: str) -> str: p = cygpath(p) elif drive: p = "/proc/cygdrive/%s/%s" % (drive.lower(), p) - p_str = os.fspath(p) # ensure it is a str and not AnyPath - return p_str.replace("\\", "/") + return p.replace("\\", "/") _cygpath_parsers: Tuple[Tuple[Pattern[str], Callable, bool], ...] = ( diff --git a/test/test_index.py b/test/test_index.py index c1db4166b..bca353748 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -579,12 +579,14 @@ def mixed_iterator(): if type_id == 0: # path (str) yield entry.path elif type_id == 1: # path (PathLike) + yield Path(entry.path) + elif type_id == 2: # path mock (PathLike) yield PathLikeMock(entry.path) - elif type_id == 2: # blob + elif type_id == 3: # blob yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) - elif type_id == 3: # BaseIndexEntry + elif type_id == 4: # BaseIndexEntry yield BaseIndexEntry(entry[:4]) - elif type_id == 4: # IndexEntry + elif type_id == 5: # IndexEntry yield entry else: raise AssertionError("Invalid Type") From 17225612969dd68cdfa6dc6b7d5ea8d1956da533 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Sat, 29 Nov 2025 13:31:22 +0000 Subject: [PATCH 1254/1392] Remove redundant str call --- git/repo/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/repo/base.py b/git/repo/base.py index e721aea40..72869c562 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -1386,7 +1386,7 @@ def _clone( proc = git.clone( multi, "--", - Git.polish_url(str(url)), + Git.polish_url(url), clone_path, with_extended_output=True, as_process=True, From 921ca8a1ddcfe84fce3d6f7f9b50a421cbee9012 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Sat, 29 Nov 2025 13:37:12 +0000 Subject: [PATCH 1255/1392] Limit mypy version due to Cygwin errors --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 75e9e81fa..552496ae4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,7 @@ coverage[toml] ddt >= 1.1.1, != 1.4.3 mock ; python_version < "3.8" -mypy +mypy<1.19.0 pre-commit pytest >= 7.3.1 pytest-cov From 12e15ba71a49652af33a8bb1556a24a19dd15c91 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Sat, 29 Nov 2025 21:27:12 +0000 Subject: [PATCH 1256/1392] Validate every fspath with tests --- git/index/base.py | 9 ++++----- git/index/fun.py | 4 ++-- git/index/typ.py | 4 ++-- git/index/util.py | 6 +++--- git/objects/submodule/base.py | 10 +++++----- git/refs/head.py | 2 -- git/refs/log.py | 3 +-- git/refs/symbolic.py | 15 ++++++++------- git/repo/base.py | 14 ++++++++------ git/repo/fun.py | 1 - git/util.py | 13 +++++++------ test/test_index.py | 8 +++++--- test/test_refs.py | 21 ++++++++++++++++++++- test/test_repo.py | 10 ++++++++++ test/test_submodule.py | 6 +++++- 15 files changed, 80 insertions(+), 46 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 2489949c1..43def2f06 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -434,7 +434,7 @@ def raise_exc(e: Exception) -> NoReturn: # characters. if abs_path not in resolved_paths: for f in self._iter_expand_paths(glob.glob(abs_path)): - yield os.fspath(f).replace(rs, "") + yield str(f).replace(rs, "") continue # END glob handling try: @@ -569,7 +569,7 @@ def resolve_blobs(self, iter_blobs: Iterator[Blob]) -> "IndexFile": for blob in iter_blobs: stage_null_key = (blob.path, 0) if stage_null_key in self.entries: - raise ValueError("Path %r already exists at stage 0" % os.fspath(blob.path)) + raise ValueError("Path %r already exists at stage 0" % str(blob.path)) # END assert blob is not stage 0 already # Delete all possible stages. @@ -652,7 +652,6 @@ def _to_relative_path(self, path: PathLike) -> PathLike: :raise ValueError: """ - path = os.fspath(path) if not osp.isabs(path): return path if self.repo.bare: @@ -660,7 +659,7 @@ def _to_relative_path(self, path: PathLike) -> PathLike: if not osp.normpath(path).startswith(os.fspath(self.repo.working_tree_dir)): raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir)) result = os.path.relpath(path, self.repo.working_tree_dir) - if path.endswith(os.sep) and not result.endswith(os.sep): + if os.fspath(path).endswith(os.sep) and not result.endswith(os.sep): result += os.sep return result @@ -1364,7 +1363,7 @@ def make_exc() -> GitCommandError: if not folder.endswith("/"): folder += "/" for entry in self.entries.values(): - if os.fspath(entry.path).startswith(folder): + if entry.path.startswith(folder): p = entry.path self._write_path_to_stdin(proc, p, p, make_exc, fprogress, read_from_stdout=False) checked_out_files.append(p) diff --git a/git/index/fun.py b/git/index/fun.py index 9832aea6b..629c19b1e 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -87,7 +87,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: return env = os.environ.copy() - env["GIT_INDEX_FILE"] = safe_decode(index.path) + env["GIT_INDEX_FILE"] = safe_decode(os.fspath(index.path)) env["GIT_EDITOR"] = ":" cmd = [hp] try: @@ -167,7 +167,7 @@ def write_cache( beginoffset = tell() write(entry.ctime_bytes) # ctime write(entry.mtime_bytes) # mtime - path_str = os.fspath(entry.path) + path_str = str(entry.path) path: bytes = force_bytes(path_str, encoding=defenc) plen = len(path) & CE_NAMEMASK # Path length assert plen == len(path), "Path %s too long to fit into index" % entry.path diff --git a/git/index/typ.py b/git/index/typ.py index 8273e5895..927633a9f 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -58,9 +58,9 @@ def __init__(self, paths: Sequence[PathLike]) -> None: def __call__(self, stage_blob: Tuple[StageType, Blob]) -> bool: blob_pathlike: PathLike = stage_blob[1].path - blob_path = Path(blob_pathlike) + blob_path: Path = blob_pathlike if isinstance(blob_pathlike, Path) else Path(blob_pathlike) for pathlike in self.paths: - path = Path(pathlike) + path: Path = pathlike if isinstance(pathlike, Path) else Path(pathlike) # TODO: Change to use `PosixPath.is_relative_to` once Python 3.8 is no # longer supported. filter_parts = path.parts diff --git a/git/index/util.py b/git/index/util.py index 80f0b014c..15eba0052 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -37,8 +37,8 @@ class TemporaryFileSwap: __slots__ = ("file_path", "tmp_file_path") def __init__(self, file_path: PathLike) -> None: - self.file_path = os.fspath(file_path) - dirname, basename = osp.split(self.file_path) + self.file_path = file_path + dirname, basename = osp.split(file_path) fd, self.tmp_file_path = tempfile.mkstemp(prefix=basename, dir=dirname) os.close(fd) with contextlib.suppress(OSError): # It may be that the source does not exist. @@ -106,7 +106,7 @@ def git_working_dir(func: Callable[..., _T]) -> Callable[..., _T]: @wraps(func) def set_git_working_dir(self: "IndexFile", *args: Any, **kwargs: Any) -> _T: cur_wd = os.getcwd() - os.chdir(os.fspath(self.repo.working_tree_dir)) + os.chdir(self.repo.working_tree_dir) try: return func(self, *args, **kwargs) finally: diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index ca6253883..36ec7c538 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -352,7 +352,7 @@ def _clone_repo( module_abspath_dir = osp.dirname(module_abspath) if not osp.isdir(module_abspath_dir): os.makedirs(module_abspath_dir) - module_checkout_path = osp.join(os.fspath(repo.working_tree_dir), path) + module_checkout_path = osp.join(repo.working_tree_dir, path) if url.startswith("../"): remote_name = cast("RemoteReference", repo.active_branch.tracking_branch()).remote_name @@ -1016,7 +1016,7 @@ def move(self, module_path: PathLike, configuration: bool = True, module: bool = return self # END handle no change - module_checkout_abspath = join_path_native(os.fspath(self.repo.working_tree_dir), module_checkout_path) + module_checkout_abspath = join_path_native(str(self.repo.working_tree_dir), module_checkout_path) if osp.isfile(module_checkout_abspath): raise ValueError("Cannot move repository onto a file: %s" % module_checkout_abspath) # END handle target files @@ -1229,7 +1229,7 @@ def remove( wtd = mod.working_tree_dir del mod # Release file-handles (Windows). gc.collect() - rmtree(wtd) + rmtree(str(wtd)) # END delete tree if possible # END handle force @@ -1313,7 +1313,7 @@ def set_parent_commit(self, commit: Union[Commit_ish, str, None], check: bool = # If check is False, we might see a parent-commit that doesn't even contain the # submodule anymore. in that case, mark our sha as being NULL. try: - self.binsha = pctree[os.fspath(self.path)].binsha + self.binsha = pctree[str(self.path)].binsha except KeyError: self.binsha = self.NULL_BIN_SHA @@ -1395,7 +1395,7 @@ def rename(self, new_name: str) -> "Submodule": destination_module_abspath = self._module_abspath(self.repo, self.path, new_name) source_dir = mod.git_dir # Let's be sure the submodule name is not so obviously tied to a directory. - if os.fspath(destination_module_abspath).startswith(os.fspath(mod.git_dir)): + if str(destination_module_abspath).startswith(str(mod.git_dir)): tmp_dir = self._module_abspath(self.repo, self.path, str(uuid.uuid4())) os.renames(source_dir, tmp_dir) source_dir = tmp_dir diff --git a/git/refs/head.py b/git/refs/head.py index 9959b889b..3c43993e7 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -8,7 +8,6 @@ __all__ = ["HEAD", "Head"] -import os from git.config import GitConfigParser, SectionConstraint from git.exc import GitCommandError from git.util import join_path @@ -45,7 +44,6 @@ class HEAD(SymbolicReference): __slots__ = () def __init__(self, repo: "Repo", path: PathLike = _HEAD_NAME) -> None: - path = os.fspath(path) if path != self._HEAD_NAME: raise ValueError("HEAD instance must point to %r, got %r" % (self._HEAD_NAME, path)) super().__init__(repo, path) diff --git a/git/refs/log.py b/git/refs/log.py index a5dfa6d20..4751cff99 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -4,7 +4,6 @@ __all__ = ["RefLog", "RefLogEntry"] from mmap import mmap -import os import os.path as osp import re import time as _time @@ -168,7 +167,7 @@ def __init__(self, filepath: Union[PathLike, None] = None) -> None: """Initialize this instance with an optional filepath, from which we will initialize our data. The path is also used to write changes back using the :meth:`write` method.""" - self._path = None if filepath is None else os.fspath(filepath) + self._path = filepath if filepath is not None: self._read_from_file() # END handle filepath diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 77e4b98f2..a422fb78c 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -4,6 +4,7 @@ __all__ = ["SymbolicReference"] import os +from pathlib import Path from gitdb.exc import BadName, BadObject @@ -76,10 +77,10 @@ class SymbolicReference: def __init__(self, repo: "Repo", path: PathLike, check_path: bool = False) -> None: self.repo = repo - self.path = os.fspath(path) + self.path: PathLike = path def __str__(self) -> str: - return self.path + return os.fspath(self.path) def __repr__(self) -> str: return '' % (self.__class__.__name__, self.path) @@ -103,7 +104,7 @@ def name(self) -> str: In case of symbolic references, the shortest assumable name is the path itself. """ - return self.path + return os.fspath(self.path) @property def abspath(self) -> PathLike: @@ -212,7 +213,7 @@ def _check_ref_name_valid(ref_path: PathLike) -> None: raise ValueError(f"Invalid reference '{ref_path}': references cannot end with a forward slash (/)") elif previous == "@" and one_before_previous is None: raise ValueError(f"Invalid reference '{ref_path}': references cannot be '@'") - elif any(component.endswith(".lock") for component in os.fspath(ref_path).split("/")): + elif any(component.endswith(".lock") for component in Path(ref_path).parts): raise ValueError( f"Invalid reference '{ref_path}': references cannot have slash-separated components that end with" " '.lock'" @@ -235,7 +236,7 @@ def _get_ref_info_helper( tokens: Union[None, List[str], Tuple[str, str]] = None repodir = _git_dir(repo, ref_path) try: - with open(os.path.join(repodir, os.fspath(ref_path)), "rt", encoding="UTF-8") as fp: + with open(os.path.join(repodir, ref_path), "rt", encoding="UTF-8") as fp: value = fp.read().rstrip() # Don't only split on spaces, but on whitespace, which allows to parse lines like: # 60b64ef992065e2600bfef6187a97f92398a9144 branch 'master' of git-server:/path/to/repo @@ -706,7 +707,7 @@ def _create( if not force and os.path.isfile(abs_ref_path): target_data = str(target) if isinstance(target, SymbolicReference): - target_data = target.path + target_data = os.fspath(target.path) if not resolve: target_data = "ref: " + target_data with open(abs_ref_path, "rb") as fd: @@ -930,4 +931,4 @@ def from_path(cls: Type[T_References], repo: "Repo", path: PathLike) -> T_Refere def is_remote(self) -> bool: """:return: True if this symbolic reference points to a remote branch""" - return self.path.startswith(self._remote_common_path_default + "/") + return os.fspath(self.path).startswith(self._remote_common_path_default + "/") diff --git a/git/repo/base.py b/git/repo/base.py index d1af79620..1f543cc57 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -126,7 +126,8 @@ class Repo: working_dir: PathLike """The working directory of the git command.""" - _working_tree_dir: Optional[str] = None + # stored as string for easier processing, but annotated as path for clearer intention + _working_tree_dir: Optional[PathLike] = None git_dir: PathLike """The ``.git`` repository directory.""" @@ -215,13 +216,13 @@ def __init__( epath = path or os.getenv("GIT_DIR") if not epath: epath = os.getcwd() + epath = os.fspath(epath) if Git.is_cygwin(): # Given how the tests are written, this seems more likely to catch Cygwin # git used from Windows than Windows git used from Cygwin. Therefore # changing to Cygwin-style paths is the relevant operation. - epath = cygpath(os.fspath(epath)) + epath = cygpath(epath) - epath = os.fspath(epath) if expand_vars and re.search(self.re_envvars, epath): warnings.warn( "The use of environment variables in paths is deprecated" @@ -306,7 +307,7 @@ def __init__( self._working_tree_dir = None # END working dir handling - self.working_dir: str = self._working_tree_dir or self.common_dir + self.working_dir: PathLike = self._working_tree_dir or self.common_dir self.git = self.GitCommandWrapperType(self.working_dir) # Special handling, in special times. @@ -366,7 +367,7 @@ def description(self, descr: str) -> None: fp.write((descr + "\n").encode(defenc)) @property - def working_tree_dir(self) -> Optional[str]: + def working_tree_dir(self) -> Optional[PathLike]: """ :return: The working tree directory of our git repository. @@ -554,7 +555,7 @@ def tag(self, path: PathLike) -> TagReference: @staticmethod def _to_full_tag_path(path: PathLike) -> str: - path_str = os.fspath(path) + path_str = str(path) if path_str.startswith(TagReference._common_path_default + "/"): return path_str if path_str.startswith(TagReference._common_default + "/"): @@ -1355,6 +1356,7 @@ def _clone( ) -> "Repo": odbt = kwargs.pop("odbt", odb_default_type) + # url may be a path and this has no effect if it is a string url = os.fspath(url) path = os.fspath(path) diff --git a/git/repo/fun.py b/git/repo/fun.py index 49097c373..3f00e60ea 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -62,7 +62,6 @@ def is_git_dir(d: PathLike) -> bool: clearly indicates that we don't support it. There is the unlikely danger to throw if we see directories which just look like a worktree dir, but are none. """ - d = os.fspath(d) if osp.isdir(d): if (osp.isdir(osp.join(d, "objects")) or "GIT_OBJECT_DIRECTORY" in os.environ) and osp.isdir( osp.join(d, "refs") diff --git a/git/util.py b/git/util.py index e82ccdcfa..c3ffdd62b 100644 --- a/git/util.py +++ b/git/util.py @@ -36,7 +36,7 @@ import logging import os import os.path as osp -import pathlib +from pathlib import Path import platform import re import shutil @@ -397,7 +397,8 @@ def _cygexpath(drive: Optional[str], path: str) -> str: p = cygpath(p) elif drive: p = "/proc/cygdrive/%s/%s" % (drive.lower(), p) - return p.replace("\\", "/") + p_str = os.fspath(p) # ensure it is a str and not AnyPath + return p_str.replace("\\", "/") _cygpath_parsers: Tuple[Tuple[Pattern[str], Callable, bool], ...] = ( @@ -464,7 +465,7 @@ def _is_cygwin_git(git_executable: str) -> bool: # Just a name given, not a real path. uname_cmd = osp.join(git_dir, "uname") - if not (pathlib.Path(uname_cmd).is_file() and os.access(uname_cmd, os.X_OK)): + if not (Path(uname_cmd).is_file() and os.access(uname_cmd, os.X_OK)): _logger.debug(f"Failed checking if running in CYGWIN: {uname_cmd} is not an executable") _is_cygwin_cache[git_executable] = is_cygwin return is_cygwin @@ -496,7 +497,7 @@ def is_cygwin_git(git_executable: Union[None, PathLike]) -> bool: elif git_executable is None: return False else: - return _is_cygwin_git(os.fspath(git_executable)) + return _is_cygwin_git(str(git_executable)) def get_user_id() -> str: @@ -522,7 +523,7 @@ def expand_path(p: PathLike, expand_vars: bool = ...) -> str: def expand_path(p: Union[None, PathLike], expand_vars: bool = True) -> Optional[PathLike]: - if isinstance(p, pathlib.Path): + if isinstance(p, Path): return p.resolve() try: p = osp.expanduser(p) # type: ignore[arg-type] @@ -1010,7 +1011,7 @@ class LockFile: __slots__ = ("_file_path", "_owns_lock") def __init__(self, file_path: PathLike) -> None: - self._file_path = os.fspath(file_path) + self._file_path = file_path self._owns_lock = False def __del__(self) -> None: diff --git a/test/test_index.py b/test/test_index.py index bca353748..33490f907 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -582,11 +582,13 @@ def mixed_iterator(): yield Path(entry.path) elif type_id == 2: # path mock (PathLike) yield PathLikeMock(entry.path) - elif type_id == 3: # blob + elif type_id == 3: # path mock in a blob yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) - elif type_id == 4: # BaseIndexEntry + elif type_id == 4: # blob + yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) + elif type_id == 5: # BaseIndexEntry yield BaseIndexEntry(entry[:4]) - elif type_id == 5: # IndexEntry + elif type_id == 6: # IndexEntry yield entry else: raise AssertionError("Invalid Type") diff --git a/test/test_refs.py b/test/test_refs.py index 08096e69e..329515807 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -25,7 +25,7 @@ import git.refs as refs from git.util import Actor -from test.lib import TestBase, with_rw_repo +from test.lib import TestBase, with_rw_repo, PathLikeMock class TestRefs(TestBase): @@ -43,6 +43,25 @@ def test_from_path(self): self.assertRaises(ValueError, TagReference, self.rorepo, "refs/invalid/tag") # Works without path check. TagReference(self.rorepo, "refs/invalid/tag", check_path=False) + # Check remoteness + assert Reference(self.rorepo, "refs/remotes/origin").is_remote() + + def test_from_pathlike(self): + # Should be able to create any reference directly. + for ref_type in (Reference, Head, TagReference, RemoteReference): + for name in ("rela_name", "path/rela_name"): + full_path = ref_type.to_full_path(PathLikeMock(name)) + instance = ref_type.from_path(self.rorepo, PathLikeMock(full_path)) + assert isinstance(instance, ref_type) + # END for each name + # END for each type + + # Invalid path. + self.assertRaises(ValueError, TagReference, self.rorepo, "refs/invalid/tag") + # Works without path check. + TagReference(self.rorepo, PathLikeMock("refs/invalid/tag"), check_path=False) + # Check remoteness + assert Reference(self.rorepo, PathLikeMock("refs/remotes/origin")).is_remote() def test_tag_base(self): tag_object_refs = [] diff --git a/test/test_repo.py b/test/test_repo.py index f089bf6b8..2a92c2523 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -15,6 +15,7 @@ import sys import tempfile from unittest import mock +from pathlib import Path import pytest @@ -369,6 +370,15 @@ def test_is_dirty_with_path(self, rwrepo): assert rwrepo.is_dirty(path="doc") is False assert rwrepo.is_dirty(untracked_files=True, path="doc") is True + @with_rw_repo("HEAD") + def test_is_dirty_with_pathlib_and_pathlike(self, rwrepo): + with open(osp.join(rwrepo.working_dir, "git", "util.py"), "at") as f: + f.write("junk") + assert rwrepo.is_dirty(path=Path("git")) is True + assert rwrepo.is_dirty(path=PathLikeMock("git")) is True + assert rwrepo.is_dirty(path=Path("doc")) is False + assert rwrepo.is_dirty(path=PathLikeMock("doc")) is False + def test_head(self): self.assertEqual(self.rorepo.head.reference.object, self.rorepo.active_branch.object) diff --git a/test/test_submodule.py b/test/test_submodule.py index edff064c4..2bf0940c9 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -28,7 +28,7 @@ from git.repo.fun import find_submodule_git_dir, touch from git.util import HIDE_WINDOWS_KNOWN_ERRORS, join_path_native, to_native_path_linux -from test.lib import TestBase, with_rw_directory, with_rw_repo +from test.lib import TestBase, with_rw_directory, with_rw_repo, PathLikeMock @contextlib.contextmanager @@ -175,6 +175,10 @@ def _do_base_tests(self, rwrepo): sma = Submodule.add(rwrepo, sm.name, sm.path) assert sma.path == sm.path + # Adding existing as pathlike + sma = Submodule.add(rwrepo, sm.name, PathLikeMock(sm.path)) + assert sma.path == sm.path + # No url and no module at path fails. self.assertRaises(ValueError, Submodule.add, rwrepo, "newsubm", "pathtorepo", url=None) From 8434967c010cc108d3a8a01d1db6d0c3971b0048 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Sat, 29 Nov 2025 21:46:54 +0000 Subject: [PATCH 1257/1392] Fix type hints --- git/index/base.py | 10 +++++----- git/index/util.py | 4 ++-- git/objects/submodule/base.py | 2 +- git/refs/symbolic.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 43def2f06..93de7933c 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -404,7 +404,7 @@ def _iter_expand_paths(self: "IndexFile", paths: Sequence[PathLike]) -> Iterator def raise_exc(e: Exception) -> NoReturn: raise e - r = os.fspath(self.repo.working_tree_dir) + r = str(self.repo.working_tree_dir) rs = r + os.sep for path in paths: abs_path = os.fspath(path) @@ -656,7 +656,7 @@ def _to_relative_path(self, path: PathLike) -> PathLike: return path if self.repo.bare: raise InvalidGitRepositoryError("require non-bare repository") - if not osp.normpath(path).startswith(os.fspath(self.repo.working_tree_dir)): + if not osp.normpath(path).startswith(str(self.repo.working_tree_dir)): raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir)) result = os.path.relpath(path, self.repo.working_tree_dir) if os.fspath(path).endswith(os.sep) and not result.endswith(os.sep): @@ -731,7 +731,7 @@ def _entries_for_paths( for path in paths: if osp.isabs(path): abspath = path - gitrelative_path = path[len(os.fspath(self.repo.working_tree_dir)) + 1 :] + gitrelative_path = path[len(str(self.repo.working_tree_dir)) + 1 :] else: gitrelative_path = path if self.repo.working_tree_dir: @@ -1036,7 +1036,7 @@ def remove( args.append("--") # Preprocess paths. - paths = self._items_to_rela_paths(items) + paths = list(map(os.fspath, self._items_to_rela_paths(items))) # type: ignore[arg-type] removed_paths = self.repo.git.rm(args, paths, **kwargs).splitlines() # Process output to gain proper paths. @@ -1363,7 +1363,7 @@ def make_exc() -> GitCommandError: if not folder.endswith("/"): folder += "/" for entry in self.entries.values(): - if entry.path.startswith(folder): + if os.fspath(entry.path).startswith(folder): p = entry.path self._write_path_to_stdin(proc, p, p, make_exc, fprogress, read_from_stdout=False) checked_out_files.append(p) diff --git a/git/index/util.py b/git/index/util.py index 15eba0052..982a5afb7 100644 --- a/git/index/util.py +++ b/git/index/util.py @@ -15,7 +15,7 @@ # typing ---------------------------------------------------------------------- -from typing import Any, Callable, TYPE_CHECKING, Optional, Type +from typing import Any, Callable, TYPE_CHECKING, Optional, Type, cast from git.types import Literal, PathLike, _T @@ -106,7 +106,7 @@ def git_working_dir(func: Callable[..., _T]) -> Callable[..., _T]: @wraps(func) def set_git_working_dir(self: "IndexFile", *args: Any, **kwargs: Any) -> _T: cur_wd = os.getcwd() - os.chdir(self.repo.working_tree_dir) + os.chdir(cast(PathLike, self.repo.working_tree_dir)) try: return func(self, *args, **kwargs) finally: diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 36ec7c538..d183672db 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -352,7 +352,7 @@ def _clone_repo( module_abspath_dir = osp.dirname(module_abspath) if not osp.isdir(module_abspath_dir): os.makedirs(module_abspath_dir) - module_checkout_path = osp.join(repo.working_tree_dir, path) + module_checkout_path = osp.join(repo.working_tree_dir, path) # type: ignore[arg-type] if url.startswith("../"): remote_name = cast("RemoteReference", repo.active_branch.tracking_branch()).remote_name diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index a422fb78c..99af4f57c 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -236,7 +236,7 @@ def _get_ref_info_helper( tokens: Union[None, List[str], Tuple[str, str]] = None repodir = _git_dir(repo, ref_path) try: - with open(os.path.join(repodir, ref_path), "rt", encoding="UTF-8") as fp: + with open(os.path.join(repodir, ref_path), "rt", encoding="UTF-8") as fp: # type: ignore[arg-type] value = fp.read().rstrip() # Don't only split on spaces, but on whitespace, which allows to parse lines like: # 60b64ef992065e2600bfef6187a97f92398a9144 branch 'master' of git-server:/path/to/repo From 171062655e24b6a6ca1a3beab3c7679278350ab5 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Mon, 1 Dec 2025 16:44:37 +0000 Subject: [PATCH 1258/1392] Add tests with non-ascii characters --- test/test_clone.py | 7 +++++++ test/test_index.py | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/test/test_clone.py b/test/test_clone.py index 143a3b51f..2d00a9e79 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -52,6 +52,13 @@ def test_clone_from_pathlike(self, rw_dir): original_repo = Repo.init(osp.join(rw_dir, "repo")) Repo.clone_from(PathLikeMock(original_repo.git_dir), PathLikeMock(os.path.join(rw_dir, "clone_pathlike"))) + @with_rw_directory + def test_clone_from_pathlike_unicode_repr(self, rw_dir): + original_repo = Repo.init(osp.join(rw_dir, "repo-áēñöưḩ̣")) + Repo.clone_from( + PathLikeMock(original_repo.git_dir), PathLikeMock(os.path.join(rw_dir, "clone_pathlike-áēñöưḩ̣")) + ) + @with_rw_directory def test_clone_from_pathlib_withConfig(self, rw_dir): original_repo = Repo.init(osp.join(rw_dir, "repo")) diff --git a/test/test_index.py b/test/test_index.py index 33490f907..dcdc3b56d 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1212,6 +1212,15 @@ def test_index_add_pathlike(self, rw_repo): rw_repo.index.add(PathLikeMock(str(file))) + @with_rw_repo("HEAD") + def test_index_add_pathlike_unicode(self, rw_repo): + git_dir = Path(rw_repo.git_dir) + + file = git_dir / "file-áēñöưḩ̣.txt" + file.touch() + + rw_repo.index.add(PathLikeMock(str(file))) + @with_rw_repo("HEAD") def test_index_add_non_normalized_path(self, rw_repo): git_dir = Path(rw_repo.git_dir) From 0cb55fb4adca4f2b26767e85ef8652ef13b834a1 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 5 Dec 2025 18:12:07 +0000 Subject: [PATCH 1259/1392] Revert "Add tests with non-ascii characters" This reverts commit 171062655e24b6a6ca1a3beab3c7679278350ab5. --- test/test_clone.py | 7 ------- test/test_index.py | 9 --------- 2 files changed, 16 deletions(-) diff --git a/test/test_clone.py b/test/test_clone.py index 2d00a9e79..143a3b51f 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -52,13 +52,6 @@ def test_clone_from_pathlike(self, rw_dir): original_repo = Repo.init(osp.join(rw_dir, "repo")) Repo.clone_from(PathLikeMock(original_repo.git_dir), PathLikeMock(os.path.join(rw_dir, "clone_pathlike"))) - @with_rw_directory - def test_clone_from_pathlike_unicode_repr(self, rw_dir): - original_repo = Repo.init(osp.join(rw_dir, "repo-áēñöưḩ̣")) - Repo.clone_from( - PathLikeMock(original_repo.git_dir), PathLikeMock(os.path.join(rw_dir, "clone_pathlike-áēñöưḩ̣")) - ) - @with_rw_directory def test_clone_from_pathlib_withConfig(self, rw_dir): original_repo = Repo.init(osp.join(rw_dir, "repo")) diff --git a/test/test_index.py b/test/test_index.py index dcdc3b56d..33490f907 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1212,15 +1212,6 @@ def test_index_add_pathlike(self, rw_repo): rw_repo.index.add(PathLikeMock(str(file))) - @with_rw_repo("HEAD") - def test_index_add_pathlike_unicode(self, rw_repo): - git_dir = Path(rw_repo.git_dir) - - file = git_dir / "file-áēñöưḩ̣.txt" - file.touch() - - rw_repo.index.add(PathLikeMock(str(file))) - @with_rw_repo("HEAD") def test_index_add_non_normalized_path(self, rw_repo): git_dir = Path(rw_repo.git_dir) From f738029ab05fe8356022248e68f9119c46b2f1e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Dec 2025 13:06:22 +0000 Subject: [PATCH 1260/1392] Bump git/ext/gitdb from `65321a2` to `4c63ee6` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `65321a2` to `4c63ee6`. - [Release notes](https://github.com/gitpython-developers/gitdb/releases) - [Commits](https://github.com/gitpython-developers/gitdb/compare/65321a28b586df60b9d1508228e2f53a35f938eb...4c63ee6636a6a3370f58b05d0bd19fec2f16dd5a) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-version: 4c63ee6636a6a3370f58b05d0bd19fec2f16dd5a dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 65321a28b..4c63ee663 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 65321a28b586df60b9d1508228e2f53a35f938eb +Subproject commit 4c63ee6636a6a3370f58b05d0bd19fec2f16dd5a From 9fa28ae108dc39cfb13282cd18d4251d0118dd52 Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 12 Dec 2025 21:41:25 +0000 Subject: [PATCH 1261/1392] Add failing tests for joining paths --- test/test_tree.py | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/test/test_tree.py b/test/test_tree.py index 7ba93bd36..dafd32847 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -8,10 +8,14 @@ from pathlib import Path import subprocess +import pytest + from git.objects import Blob, Tree +from git.repo import Repo from git.util import cwd from test.lib import TestBase, with_rw_directory +from .lib.helper import PathLikeMock, with_rw_repo class TestTree(TestBase): @@ -161,3 +165,57 @@ def lib_folder(t, _d): assert root[item.path] == item == root / item.path # END for each item assert found_slash + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_string_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / ".gitignore" + assert isinstance(blob, Blob) + assert blob.hexsha == "787b3d442a113b78e343deb585ab5531eb7187fa" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_pathlike_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / PathLikeMock(".gitignore") + assert isinstance(blob, Blob) + assert blob.hexsha == "787b3d442a113b78e343deb585ab5531eb7187fa" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_invalid_string_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + with pytest.raises(KeyError): + repo.tree() / "doesnotexist" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_invalid_pathlike_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + with pytest.raises(KeyError): + repo.tree() / PathLikeMock("doesnotexist") + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_nested_string_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / "git/__init__.py" + assert isinstance(blob, Blob) + assert blob.hexsha == "d87dcbdbb65d2782e14eea27e7f833a209c052f3" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_nested_pathlike_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / PathLikeMock("git/__init__.py") + assert isinstance(blob, Blob) + assert blob.hexsha == "d87dcbdbb65d2782e14eea27e7f833a209c052f3" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_folder_string_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / "git" + assert isinstance(blob, Tree) + assert blob.hexsha == "ec8ae429156d65afde4bbb3455570193b56f0977" + + @with_rw_repo("0.3.2.1") + def test_repo_lookup_folder_pathlike_path(self, rw_repo): + repo = Repo(rw_repo.git_dir) + blob = repo.tree() / PathLikeMock("git") + assert isinstance(blob, Tree) + assert blob.hexsha == "ec8ae429156d65afde4bbb3455570193b56f0977" From 88e26141c738f6ac3beb1a433039611f88c2c30d Mon Sep 17 00:00:00 2001 From: George Ogden Date: Fri, 12 Dec 2025 21:41:50 +0000 Subject: [PATCH 1262/1392] Allow joining path to tree --- git/objects/tree.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/git/objects/tree.py b/git/objects/tree.py index 1845d0d0d..a3d611c80 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -5,6 +5,7 @@ __all__ = ["TreeModifier", "Tree"] +import os import sys import git.diff as git_diff @@ -230,7 +231,7 @@ def _iter_convert_to_object(self, iterable: Iterable[TreeCacheTup]) -> Iterator[ raise TypeError("Unknown mode %o found in tree data for path '%s'" % (mode, path)) from e # END for each item - def join(self, file: str) -> IndexObjUnion: + def join(self, file: PathLike) -> IndexObjUnion: """Find the named object in this tree's contents. :return: @@ -241,6 +242,7 @@ def join(self, file: str) -> IndexObjUnion: If the given file or tree does not exist in this tree. """ msg = "Blob or Tree named %r not found" + file = os.fspath(file) if "/" in file: tree = self item = self @@ -269,7 +271,7 @@ def join(self, file: str) -> IndexObjUnion: raise KeyError(msg % file) # END handle long paths - def __truediv__(self, file: str) -> IndexObjUnion: + def __truediv__(self, file: PathLike) -> IndexObjUnion: """The ``/`` operator is another syntax for joining. See :meth:`join` for details. From c8b58c09904dabe67222165e4d3eecf4c8f07490 Mon Sep 17 00:00:00 2001 From: George Ogden <38294960+George-Ogden@users.noreply.github.com> Date: Sun, 14 Dec 2025 17:42:21 +0000 Subject: [PATCH 1263/1392] Update test/test_tree.py Rename blob to tree Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- test/test_tree.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_tree.py b/test/test_tree.py index dafd32847..629fd4d32 100644 --- a/test/test_tree.py +++ b/test/test_tree.py @@ -209,13 +209,13 @@ def test_repo_lookup_nested_pathlike_path(self, rw_repo): @with_rw_repo("0.3.2.1") def test_repo_lookup_folder_string_path(self, rw_repo): repo = Repo(rw_repo.git_dir) - blob = repo.tree() / "git" - assert isinstance(blob, Tree) - assert blob.hexsha == "ec8ae429156d65afde4bbb3455570193b56f0977" + tree = repo.tree() / "git" + assert isinstance(tree, Tree) + assert tree.hexsha == "ec8ae429156d65afde4bbb3455570193b56f0977" @with_rw_repo("0.3.2.1") def test_repo_lookup_folder_pathlike_path(self, rw_repo): repo = Repo(rw_repo.git_dir) - blob = repo.tree() / PathLikeMock("git") - assert isinstance(blob, Tree) - assert blob.hexsha == "ec8ae429156d65afde4bbb3455570193b56f0977" + tree = repo.tree() / PathLikeMock("git") + assert isinstance(tree, Tree) + assert tree.hexsha == "ec8ae429156d65afde4bbb3455570193b56f0977" From 9e24eb6b72c1851e46e09133b83b48f2059037d7 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 1 Jan 2026 16:32:19 +0100 Subject: [PATCH 1264/1392] Prepare next release --- VERSION | 2 +- doc/source/changes.rst | 34 ++++++++++++++++++++-------------- git/ext/gitdb | 2 +- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/VERSION b/VERSION index 3c91929a4..fd84d1e83 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.45 +3.1.46 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 151059ed2..9b82e7513 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ========= +3.1.46 +====== + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.46 + 3.1.45 ====== @@ -111,7 +117,7 @@ https://github.com/gitpython-developers/gitpython/milestone/61?closed=1 but a necessary fix for https://github.com/gitpython-developers/GitPython/issues/1515. Please take a look at the PR for more information and how to bypass these protections in case they cause breakage: https://github.com/gitpython-developers/GitPython/pull/1521. - + See the following for all changes. https://github.com/gitpython-developers/gitpython/milestone/60?closed=1 @@ -176,38 +182,38 @@ https://github.com/gitpython-developers/gitpython/milestone/53?closed=1 * General: - Remove python 3.6 support - + - Remove distutils ahead of deprecation in standard library. - + - Update sphinx to 4.1.12 and use autodoc-typehints. - + - Include README as long_description on PyPI - + - Test against earliest and latest minor version available on Github Actions (e.g. 3.9.0 and 3.9.7) - + * Typing: - Add types to ALL functions. - + - Ensure py.typed is collected. - + - Increase mypy strictness with disallow_untyped_defs, warn_redundant_casts, warn_unreachable. - + - Use typing.NamedTuple and typing.OrderedDict now 3.6 dropped. - + - Make Protocol classes ABCs at runtime due to new behaviour/bug in 3.9.7 & 3.10.0-rc1 - + - Remove use of typing.TypeGuard until later release, to allow dependent libs time to update. - + - Tracking issue: https://github.com/gitpython-developers/GitPython/issues/1095 * Runtime improvements: - Add clone_multi_options support to submodule.add() - + - Delay calling get_user_id() unless essential, to support sand-boxed environments. - + - Add timeout to handle_process_output(), in case thread.join() hangs. See the following for details: diff --git a/git/ext/gitdb b/git/ext/gitdb index 4c63ee663..335c0f661 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 4c63ee6636a6a3370f58b05d0bd19fec2f16dd5a +Subproject commit 335c0f66173eecdc7b2597c2b6c3d1fde795df30 From 0b45d122c68cd7de9e0fbcd9f6cc336bee9371fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:05:40 +0000 Subject: [PATCH 1265/1392] Bump git/ext/gitdb from `335c0f6` to `4c63ee6` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `335c0f6` to `4c63ee6`. - [Release notes](https://github.com/gitpython-developers/gitdb/releases) - [Commits](https://github.com/gitpython-developers/gitdb/compare/335c0f66173eecdc7b2597c2b6c3d1fde795df30...4c63ee6636a6a3370f58b05d0bd19fec2f16dd5a) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-version: 4c63ee6636a6a3370f58b05d0bd19fec2f16dd5a dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 335c0f661..4c63ee663 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 335c0f66173eecdc7b2597c2b6c3d1fde795df30 +Subproject commit 4c63ee6636a6a3370f58b05d0bd19fec2f16dd5a From acc6a8cc83188b107344797cdf6b88398fef0676 Mon Sep 17 00:00:00 2001 From: Ilyas Timour Date: Sat, 10 Jan 2026 11:07:29 +0100 Subject: [PATCH 1266/1392] DOC: README Add urls and updated a relative url --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 59c6f995b..412d38205 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ by setting the `GIT_PYTHON_GIT_EXECUTABLE=` environment variable. - Git (1.7.x or newer) - Python >= 3.7 -The list of dependencies are listed in `./requirements.txt` and `./test-requirements.txt`. +The list of dependencies are listed in [`./requirements.txt`](https://github.com/gitpython-developers/GitPython/blob/main/requirements.txt) and [`./test-requirements.txt`](https://github.com/gitpython-developers/GitPython/blob/main/test-requirements.txt). The installer takes care of installing them for you. ### INSTALL @@ -180,7 +180,7 @@ Style and formatting checks, and running tests on all the different supported Py #### Configuration files -Specific tools are all configured in the `./pyproject.toml` file: +Specific tools are all configured in the [`./pyproject.toml`](https://github.com/gitpython-developers/GitPython/blob/main/pyproject.toml) file: - `pytest` (test runner) - `coverage.py` (code coverage) @@ -189,9 +189,9 @@ Specific tools are all configured in the `./pyproject.toml` file: Orchestration tools: -- Configuration for `pre-commit` is in the `./.pre-commit-config.yaml` file. -- Configuration for `tox` is in `./tox.ini`. -- Configuration for GitHub Actions (CI) is in files inside `./.github/workflows/`. +- Configuration for `pre-commit` is in the [`./.pre-commit-config.yaml`](https://github.com/gitpython-developers/GitPython/blob/main/.pre-commit-config.yaml) file. +- Configuration for `tox` is in [`./tox.ini`](https://github.com/gitpython-developers/GitPython/blob/main/tox.ini). +- Configuration for GitHub Actions (CI) is in files inside [`./.github/workflows/`](https://github.com/gitpython-developers/GitPython/tree/main/.github/workflows). ### Contributions @@ -212,8 +212,8 @@ Please have a look at the [contributions file][contributing]. ### How to make a new release -1. Update/verify the **version** in the `VERSION` file. -2. Update/verify that the `doc/source/changes.rst` changelog file was updated. It should include a link to the forthcoming release page: `https://github.com/gitpython-developers/GitPython/releases/tag/` +1. Update/verify the **version** in the [`VERSION`](https://github.com/gitpython-developers/GitPython/blob/main/VERSION) file. +2. Update/verify that the [`doc/source/changes.rst`](https://github.com/gitpython-developers/GitPython/blob/main/doc/source/changes.rst) changelog file was updated. It should include a link to the forthcoming release page: `https://github.com/gitpython-developers/GitPython/releases/tag/` 3. Commit everything. 4. Run `git tag -s ` to tag the version in Git. 5. _Optionally_ create and activate a [virtual environment](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment). (Then the next step can install `build` and `twine`.) @@ -240,7 +240,7 @@ Please have a look at the [contributions file][contributing]. [3-Clause BSD License](https://opensource.org/license/bsd-3-clause/), also known as the New BSD License. See the [LICENSE file][license]. -One file exclusively used for fuzz testing is subject to [a separate license, detailed here](./fuzzing/README.md#license). +One file exclusively used for fuzz testing is subject to [a separate license, detailed here](https://github.com/gitpython-developers/GitPython/blob/main/fuzzing/README.md#license). This file is not included in the wheel or sdist packages published by the maintainers of GitPython. [contributing]: https://github.com/gitpython-developers/GitPython/blob/main/CONTRIBUTING.md From 1a74bce18b5492a3cec9b839a4cb706d78eca466 Mon Sep 17 00:00:00 2001 From: danielyan Date: Mon, 9 Feb 2026 20:03:39 +0000 Subject: [PATCH 1267/1392] Fix GitConfigParser ignoring multiple [include] path entries When an [include] section has multiple entries with the same key (e.g. multiple 'path' values), only the last one was respected. This is because _included_paths() used self.items(section) which delegates to _OMD.items(), and _OMD.__getitem__ returns only the last value for a given key. Fix by using _OMD.items_all() to retrieve all values for each key in the include/includeIf sections, ensuring all paths are processed. Fixes gitpython-developers#2099 --- git/config.py | 18 ++++++++++++++---- test/test_config.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/git/config.py b/git/config.py index 769929441..c6eaf8f7b 100644 --- a/git/config.py +++ b/git/config.py @@ -549,11 +549,21 @@ def _included_paths(self) -> List[Tuple[str, str]]: :return: The list of paths, where each path is a tuple of (option, value). """ + + def _all_items(section: str) -> List[Tuple[str, str]]: + """Return all (key, value) pairs for a section, including duplicate keys.""" + return [ + (key, value) + for key, values in self._sections[section].items_all() + if key != "__name__" + for value in values + ] + paths = [] for section in self.sections(): if section == "include": - paths += self.items(section) + paths += _all_items(section) match = CONDITIONAL_INCLUDE_REGEXP.search(section) if match is None or self._repo is None: @@ -579,7 +589,7 @@ def _included_paths(self) -> List[Tuple[str, str]]: ) if self._repo.git_dir: if fnmatch.fnmatchcase(os.fspath(self._repo.git_dir), value): - paths += self.items(section) + paths += _all_items(section) elif keyword == "onbranch": try: @@ -589,11 +599,11 @@ def _included_paths(self) -> List[Tuple[str, str]]: continue if fnmatch.fnmatchcase(branch_name, value): - paths += self.items(section) + paths += _all_items(section) elif keyword == "hasconfig:remote.*.url": for remote in self._repo.remotes: if fnmatch.fnmatchcase(remote.url, value): - paths += self.items(section) + paths += _all_items(section) break return paths diff --git a/test/test_config.py b/test/test_config.py index 56ac0f304..11ea52d16 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -246,6 +246,43 @@ def check_test_value(cr, value): with GitConfigParser(fpa, read_only=True) as cr: check_test_value(cr, tv) + @with_rw_directory + def test_multiple_include_paths_with_same_key(self, rw_dir): + """Test that multiple 'path' entries under [include] are all respected. + + Regression test for https://github.com/gitpython-developers/GitPython/issues/2099. + Git config allows multiple ``path`` values under ``[include]``, e.g.:: + + [include] + path = file1 + path = file2 + + Previously only one of these was included because _OMD.items() returns + only the last value for each key. + """ + # Create two config files to be included. + fp_inc1 = osp.join(rw_dir, "inc1.cfg") + fp_inc2 = osp.join(rw_dir, "inc2.cfg") + fp_main = osp.join(rw_dir, "main.cfg") + + with GitConfigParser(fp_inc1, read_only=False) as cw: + cw.set_value("user", "name", "from-inc1") + + with GitConfigParser(fp_inc2, read_only=False) as cw: + cw.set_value("core", "bar", "from-inc2") + + # Write a config with two path entries under a single [include] section. + # We write it manually because set_value would overwrite the key. + with open(fp_main, "w") as f: + f.write("[include]\n") + f.write(f"\tpath = {fp_inc1}\n") + f.write(f"\tpath = {fp_inc2}\n") + + with GitConfigParser(fp_main, read_only=True) as cr: + # Both included files should be loaded. + assert cr.get_value("user", "name") == "from-inc1" + assert cr.get_value("core", "bar") == "from-inc2" + @pytest.mark.xfail( sys.platform == "win32", reason='Second config._has_includes() assertion fails (for "config is included if path is matching git_dir")', From 5eaa310e6a03d146892a95ef596cf3688b875364 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Mar 2026 02:54:30 +0000 Subject: [PATCH 1268/1392] Initial plan From e4ad410ac3baf2046bd4043394e7cbb119045cc1 Mon Sep 17 00:00:00 2001 From: "chatgpt-codex-connector[bot]" <199175422+chatgpt-codex-connector[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 02:57:28 +0000 Subject: [PATCH 1269/1392] Bump smmap version to 5.0.3 for Python 3.13 metadata release Co-authored-by: Sebastian Thiel --- doc/source/changes.rst | 8 ++++++++ smmap/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index dc243eb0c..faed6085d 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,14 @@ Changelog ######### +****** +v5.0.3 +****** + +- declare support for Python 3.13 + +For more, see https://github.com/gitpython-developers/smmap/compare/v5.0.3...v5.0.2 + ****** v5.0.2 ****** diff --git a/smmap/__init__.py b/smmap/__init__.py index 657c8c5af..623181034 100644 --- a/smmap/__init__.py +++ b/smmap/__init__.py @@ -3,7 +3,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" __homepage__ = "https://github.com/gitpython-developers/smmap" -version_info = (5, 0, 2) +version_info = (5, 0, 3) __version__ = '.'.join(str(i) for i in version_info) # make everything available in root package for convenience From af034fceb98c27f2f57091365eb62353ec7a354c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 11:10:26 +0000 Subject: [PATCH 1270/1392] Bump gitdb/ext/smmap from `5ec977a` to `e4ad410` Bumps [gitdb/ext/smmap](https://github.com/gitpython-developers/smmap) from `5ec977a` to `e4ad410`. - [Release notes](https://github.com/gitpython-developers/smmap/releases) - [Commits](https://github.com/gitpython-developers/smmap/compare/5ec977a3b280e5dccb40cb20eba56ea26a84bd48...e4ad410ac3baf2046bd4043394e7cbb119045cc1) --- updated-dependencies: - dependency-name: gitdb/ext/smmap dependency-version: e4ad410ac3baf2046bd4043394e7cbb119045cc1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- gitdb/ext/smmap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitdb/ext/smmap b/gitdb/ext/smmap index 5ec977a3b..e4ad410ac 160000 --- a/gitdb/ext/smmap +++ b/gitdb/ext/smmap @@ -1 +1 @@ -Subproject commit 5ec977a3b280e5dccb40cb20eba56ea26a84bd48 +Subproject commit e4ad410ac3baf2046bd4043394e7cbb119045cc1 From 2f6e5441b323a7fbde672e3df30e564628d43371 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Mar 2026 12:32:24 -0400 Subject: [PATCH 1271/1392] Switch back from Alpine to Debian for WSL For #2107. Note that this does not affect the container workflow `alpine-test.yml` workflow (that is, it doesn't affect actually running the test suite in Alpine Linux, which continues to work), only WSL in Windows jobs in the main workflow `pythonpackage.yml`. --- .github/workflows/pythonpackage.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index ac764d9a7..6c5cf4552 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -54,8 +54,7 @@ jobs: uses: Vampire/setup-wsl@v6.0.0 with: wsl-version: 1 - distribution: Alpine - additional-packages: bash + distribution: Debian - name: Prepare this repo for tests run: | From 9648077f1913b41498b55f253a5188b4ba4b27dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:58:20 +0000 Subject: [PATCH 1272/1392] Bump git/ext/gitdb from `4c63ee6` to `5c1b303` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `4c63ee6` to `5c1b303`. - [Release notes](https://github.com/gitpython-developers/gitdb/releases) - [Commits](https://github.com/gitpython-developers/gitdb/compare/4c63ee6636a6a3370f58b05d0bd19fec2f16dd5a...5c1b3036a6e34782e0ab6ce85e5ae64fe777fdbe) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-version: 5c1b3036a6e34782e0ab6ce85e5ae64fe777fdbe dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 4c63ee663..5c1b3036a 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 4c63ee6636a6a3370f58b05d0bd19fec2f16dd5a +Subproject commit 5c1b3036a6e34782e0ab6ce85e5ae64fe777fdbe From 98b78d2c1558e4d2fef1d52ecfd30c15f181c38b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Mar 2026 13:27:16 -0400 Subject: [PATCH 1273/1392] Run `gc.collect()` twice in `test_rename` on Python 3.12 Recently, the conditional `gc.collect()` step for Python >= 3.12 in `TestSubmodule.test_rename` is often insufficient. This has mainly been seen in #2248. For example: https://github.com/gitpython-developers/GitPython/actions/runs/22864869684/job/66331124651?pr=2106#step:12:620 In principle, there can be situations with finalizers where a cycle is only collectable due to finalization that happened due to a previous collection. Therefore, there is occasionally a benefit to collecting twice. This does that, in the hope that it will help. --- test/test_submodule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_submodule.py b/test/test_submodule.py index 2bf0940c9..47647f2a1 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -1011,6 +1011,7 @@ def test_rename(self, rwdir): # garbage collector detailed in https://github.com/python/cpython/issues/97922.) if sys.platform == "win32" and sys.version_info >= (3, 12): gc.collect() + gc.collect() # Some finalizer scenarios need two collections, at least in theory. new_path = "renamed/myname" assert sm.move(new_path).name == new_path From 3cfd22938581631cc22f80ea751eecea1747a1d8 Mon Sep 17 00:00:00 2001 From: Luca Weyrich Date: Wed, 25 Feb 2026 09:04:29 +0100 Subject: [PATCH 1274/1392] fix: ignore AutoInterrupt terminate errors during shutdown --- git/cmd.py | 8 ++++++-- test/test_autointerrupt.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 test/test_autointerrupt.py diff --git a/git/cmd.py b/git/cmd.py index 15d7820df..78a9f4c78 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -368,8 +368,12 @@ def _terminate(self) -> None: status = proc.wait() # Ensure the process goes away. self.status = self._status_code_if_terminate or status - except OSError as ex: - _logger.info("Ignored error after process had died: %r", ex) + except (OSError, AttributeError) as ex: + # On interpreter shutdown (notably on Windows), parts of the stdlib used by + # subprocess can already be torn down (e.g. `subprocess._winapi` becomes None), + # which can cause AttributeError during terminate(). In that case, we prefer + # to silently ignore to avoid noisy "Exception ignored in: __del__" messages. + _logger.info("Ignored error while terminating process: %r", ex) # END exception handling def __del__(self) -> None: diff --git a/test/test_autointerrupt.py b/test/test_autointerrupt.py new file mode 100644 index 000000000..56e101efb --- /dev/null +++ b/test/test_autointerrupt.py @@ -0,0 +1,35 @@ +import pytest + +from git.cmd import Git + + +class _DummyProc: + """Minimal stand-in for subprocess.Popen used to exercise AutoInterrupt. + + We deliberately raise AttributeError from terminate() to simulate interpreter + shutdown on Windows where subprocess internals (e.g. subprocess._winapi) may + already be torn down. + """ + + stdin = None + stdout = None + stderr = None + + def poll(self): + return None + + def terminate(self): + raise AttributeError("TerminateProcess") + + def wait(self): # pragma: no cover - should not be reached in this test + raise AssertionError("wait() should not be called if terminate() fails") + + +def test_autointerrupt_terminate_ignores_attributeerror(): + ai = Git.AutoInterrupt(_DummyProc(), args=["git", "rev-list"]) + + # Should not raise, even if terminate() triggers AttributeError. + ai._terminate() + + # Ensure the reference is cleared to avoid repeated attempts. + assert ai.proc is None From 357aad19912087b9819844acb3b904cf3c23b29e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Mar 2026 13:16:43 -0400 Subject: [PATCH 1275/1392] Remove unnecessary `pytest` import in a test module --- test/test_autointerrupt.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_autointerrupt.py b/test/test_autointerrupt.py index 56e101efb..645ec402c 100644 --- a/test/test_autointerrupt.py +++ b/test/test_autointerrupt.py @@ -1,5 +1,3 @@ -import pytest - from git.cmd import Git From 078f3514f5b70052c0126e03713f1e543b26e8c3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Mar 2026 14:29:43 -0400 Subject: [PATCH 1276/1392] Run the the `pre-commit` CI job on `ubuntu-slim` The `ubuntu-slim` runner is lighter weight, being a container rather than using a whole VM, and having only one vCPU, less RAM, and a 15 minute time limit. It's not suitable for most of our CI jobs in GitPython, but it should work well for our `pre-commit` checks. (If it doesn't, that's reason to suspect they might be better removed from `pre-commit` and run in a different way.) - https://github.blog/changelog/2026-01-22-1-vcpu-linux-runner-now-generally-available-in-github-actions/ - https://github.com/actions/runner-images/blob/main/images/ubuntu-slim/ubuntu-slim-Readme.md --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 956b38963..e32e946c8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,7 +7,7 @@ permissions: jobs: lint: - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - uses: actions/checkout@v6 From 35db8094038a817780f614a518663362aec11a4c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Mar 2026 14:38:42 -0400 Subject: [PATCH 1277/1392] Keep `pre-commit` hooks up to date using Dependabot - Add `pre-commit` as an ecosystem for Dependabot version updates, now that it is available as a beta ecosystem. Enable beta ecosystems to allow this. - Group the updates and use a monthly cadence to avoid getting swamped by frequent automated PRs. - It would be valuable in the future to Use a 7-day cooldown period rather than taking new versions immediately once released. (This may also be of value to developers who use `pre-commit` locally.) However, this doesn't do that, since the Dependabot ecosystem for `pre-commit` does not currently support `cooldown`. - Use a less busy style (less unnecessary quoting) than was being used in `dependabot.yml` before, since this new stanza is more elaborate than before. Apply that style to the existing stanzas for consistency. --- .github/dependabot.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2fe73ca77..16d5f11bc 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,11 +1,20 @@ version: 2 +enable-beta-ecosystems: true updates: -- package-ecosystem: "github-actions" +- package-ecosystem: github-actions directory: "/" schedule: - interval: "weekly" + interval: weekly -- package-ecosystem: "gitsubmodule" +- package-ecosystem: gitsubmodule directory: "/" schedule: - interval: "weekly" + interval: weekly + +- package-ecosystem: pre-commit + directory: "/" + schedule: + interval: monthly + groups: + pre-commit: + patterns: ["*"] From a4bc27a7ae6b74df397de3f15fd3ee97fa06d28c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 19:48:14 +0000 Subject: [PATCH 1278/1392] Bump the pre-commit group with 5 updates Bumps the pre-commit group with 5 updates: | Package | From | To | | --- | --- | --- | | [https://github.com/codespell-project/codespell](https://github.com/codespell-project/codespell) | `v2.4.1` | `2.4.2` | | [https://github.com/astral-sh/ruff-pre-commit](https://github.com/astral-sh/ruff-pre-commit) | `v0.11.12` | `0.15.5` | | [https://github.com/shellcheck-py/shellcheck-py](https://github.com/shellcheck-py/shellcheck-py) | `v0.10.0.1` | `0.11.0.1` | | [https://github.com/pre-commit/pre-commit-hooks](https://github.com/pre-commit/pre-commit-hooks) | `v5.0.0` | `6.0.0` | | [https://github.com/abravalheri/validate-pyproject](https://github.com/abravalheri/validate-pyproject) | `v0.24.1` | `0.25` | Updates `https://github.com/codespell-project/codespell` from v2.4.1 to 2.4.2 - [Release notes](https://github.com/codespell-project/codespell/releases) - [Commits](https://github.com/codespell-project/codespell/compare/v2.4.1...v2.4.2) Updates `https://github.com/astral-sh/ruff-pre-commit` from v0.11.12 to 0.15.5 - [Release notes](https://github.com/astral-sh/ruff-pre-commit/releases) - [Commits](https://github.com/astral-sh/ruff-pre-commit/compare/v0.11.12...v0.15.5) Updates `https://github.com/shellcheck-py/shellcheck-py` from v0.10.0.1 to 0.11.0.1 - [Commits](https://github.com/shellcheck-py/shellcheck-py/compare/v0.10.0.1...v0.11.0.1) Updates `https://github.com/pre-commit/pre-commit-hooks` from v5.0.0 to 6.0.0 - [Release notes](https://github.com/pre-commit/pre-commit-hooks/releases) - [Changelog](https://github.com/pre-commit/pre-commit-hooks/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit-hooks/compare/v5.0.0...v6.0.0) Updates `https://github.com/abravalheri/validate-pyproject` from v0.24.1 to 0.25 - [Release notes](https://github.com/abravalheri/validate-pyproject/releases) - [Changelog](https://github.com/abravalheri/validate-pyproject/blob/main/CHANGELOG.rst) - [Commits](https://github.com/abravalheri/validate-pyproject/compare/v0.24.1...v0.25) --- updated-dependencies: - dependency-name: https://github.com/codespell-project/codespell dependency-version: 2.4.2 dependency-type: direct:production dependency-group: pre-commit - dependency-name: https://github.com/astral-sh/ruff-pre-commit dependency-version: 0.15.5 dependency-type: direct:production dependency-group: pre-commit - dependency-name: https://github.com/shellcheck-py/shellcheck-py dependency-version: 0.11.0.1 dependency-type: direct:production dependency-group: pre-commit - dependency-name: https://github.com/pre-commit/pre-commit-hooks dependency-version: 6.0.0 dependency-type: direct:production dependency-group: pre-commit - dependency-name: https://github.com/abravalheri/validate-pyproject dependency-version: '0.25' dependency-type: direct:production dependency-group: pre-commit ... Signed-off-by: dependabot[bot] --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 737b56d45..3bd9cbce9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ repos: - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 + rev: v2.4.2 hooks: - id: codespell additional_dependencies: [tomli] exclude: ^test/fixtures/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.12 + rev: v0.15.5 hooks: - id: ruff-check args: ["--fix"] @@ -16,14 +16,14 @@ repos: exclude: ^git/ext/ - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.10.0.1 + rev: v0.11.0.1 hooks: - id: shellcheck args: [--color] exclude: ^test/fixtures/polyglot$|^git/ext/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: end-of-file-fixer exclude: ^test/fixtures/|COPYING|LICENSE @@ -33,6 +33,6 @@ repos: - id: check-merge-conflict - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.24.1 + rev: v0.25 hooks: - id: validate-pyproject From d1ab2e40b3bd03f84dcf440ed920bf7ab512366f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Mar 2026 14:17:27 -0400 Subject: [PATCH 1279/1392] Test free-threaded interpreter on macOS As discussed in #2005 and #2011, we had not been doing this before. Conditions have changed in two relevant ways: - The free-threaded interpreter has been around longer and it sees more use. - The macOS runners are very fast now. The specific motivations for doing this now are: - In view of the condition described in #2109 and how the change there seems to have helped with it, there's some reason to think *patch* versions of Python sometimes affect GitPython in ways it makes possibly unfounded assumptions about the effect of garbage collection. This mainly affects Windows and it is not specific to free-threaded builds. However, in principle we could also see assumptions violated in tests we think always work on Unix-like operating systems, due to differences in how garbage collection works in free-threaded interpreters. Therefore, the assumption that this only needs to be tested occasionally is not as well founded I assumed when I suggested testing it only on GNU/Linux. - We may add 3.14 jobs to CI soon, and it's useful to be able to see how both free-threaded interpreters work on CI, as well as to confirm for at least a short while that they are continuing to work as expected. This macOS free-threaded interpreter CI jobs could be disabled once more if necessary, or if they're found to make CI complete slower in PRs by even a small amount so long as they don't seem to be surfacing anything. --- .github/workflows/pythonpackage.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 6c5cf4552..6c1f7a67a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -17,8 +17,6 @@ jobs: exclude: - os-type: macos python-version: "3.7" # Not available for the ARM-based macOS runners. - - os-type: macos - python-version: "3.13t" - os-type: windows python-version: "3.13" # FIXME: Fix and enable Python 3.13 on Windows (#1955). - os-type: windows From 53c0a8800bc4297c2b50c14e8ada45bdaedfe2ab Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Mar 2026 15:15:28 -0400 Subject: [PATCH 1280/1392] Test Python 3.14 on Ubuntu and macOS on CI The status of 3.14 is now effectively the same as the status of 3.13 when we started testing it in #1990. This doesn't enable it on Windows yet, for the same reason that we still have not yet enabled regular CI tests of 3.13 on Windows. It is hoped that 3.13 and 3.14 can be gotten fully working on Windows (rather than just mostly working, we think) soon; these exclusions are meant to be temporary. Both the usual GIL interpreter and the free-threaded (nogil) intepreters are tested. See the immediately preceding commit on the tradeoffs involved in doing so. --- .github/workflows/pythonpackage.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 6c1f7a67a..3d2cb9b63 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -13,14 +13,18 @@ jobs: strategy: matrix: os-type: [ubuntu, macos, windows] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t", "3.14", "3.14t"] exclude: - os-type: macos python-version: "3.7" # Not available for the ARM-based macOS runners. - os-type: windows - python-version: "3.13" # FIXME: Fix and enable Python 3.13 on Windows (#1955). + python-version: "3.13" # FIXME: Fix and enable Python 3.13 and 3.14 on Windows (#1955). - os-type: windows python-version: "3.13t" + - os-type: windows + python-version: "3.14" + - os-type: windows + python-version: "3.14t" include: - os-ver: latest - os-type: ubuntu From d1ca2af85b96062a9d5979edf895ddf3968cfa49 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Mar 2026 17:42:21 -0400 Subject: [PATCH 1281/1392] Upgrade Sphinx to ~7.4.7 Except on Python 3.8, where 7.1.2 is the latest compatible version. (This would also apply to versions lower than 3.8, but we don't support building docs on any such versions, even though we still support installing and using GitPython on 3.7.) The reason for this change is that, starting in Python 3.14, the `ast` module no longer has a `Str` member. String literals are instead represented by `ast.Constant` (and the type of the value can be checked to see if it's a string). But versions of `sphinx` lower than 7.2.0 rely on `ast.Str` being present. This causes our documentation not to be able to build at all starting in 3.14. The most important part of the error is: Exception occurred: File "/opt/hostedtoolcache/Python/3.14.3/x64/lib/python3.14/site-packages/sphinx/pycode/__init__.py", line 141, in analyze raise PycodeError(f'parsing {self.srcname!r} failed: {exc!r}') from exc sphinx.errors.PycodeError: parsing '/home/runner/work/GitPython/GitPython/git/index/base.py' failed: AttributeError("module 'ast' has no attribute 'Str'") An example of code in `sphinx` 7.1.2 that will cause such an error is `sphinx.pycode.parser.visit_Expr` implementation, which starts: if (isinstance(self.previous, (ast.Assign, ast.AnnAssign)) and isinstance(node.value, ast.Str)): In `sphinx` 7.2.0, `sphinx.pycode.parser.visit_Expr` instead begins: if (isinstance(self.previous, (ast.Assign, ast.AnnAssign)) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str)): This upgrades `sphinx` on all versions of Python where it *can* be installed at a version that has such changes -- rather than only on Python 3.14 -- for consistency, including consistency in possible minor variations in generated documentation that could otherwise arise from using different versions of `sphinx` unnecessarily. As for why this upgrades to 7.4.7 rather than only to 7.2.0, that's because they are both compatible with the same versions of Python, and as far as I know there's no reason to prefer an earlier version within that range. Although GitPython still supports being installed and run on Python 3.8 (and even on Python 3.7), it has been end-of-life (i.e., no longer supported by the Python Software Foundation) for quite some time now. That the version of Sphinx used to build documentation will now be different on Python 3.8 than other versions is a reason not to use Python 3.8 for this purpose, but probablly already not the most important reason. The change here is conceptually similar to, but much simpler than, the change in #1954, which upgraded Sphinx to 7.1.2 on all Python versions GitPython suppports other than Python 3.7. The subsequent change in #1956 of removing support for building the GitPython documentation on Python 3.7 may be paralleled for 3.8 shortly. --- doc/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 81140d898..cbf34cc69 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,3 +1,4 @@ -sphinx >= 7.1.2, < 7.2 +sphinx >= 7.4.7, < 8 ; python_version >= "3.9" +sphinx >= 7.1.2, < 7.2 ; python_version < "3.9" sphinx_rtd_theme sphinx-autodoc-typehints From 8d979061850e8659a089e4249be84d906c409334 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Mar 2026 18:10:49 -0400 Subject: [PATCH 1282/1392] Don't support building documentation on Python 3.8 This discontinues supporting building documentation on Python 3.8. It does not affect installing or running GitPython on Python 3.8 (except when the `doc` extra is used, but this is only used for building documentation). The reason is that it is no longer possible to use the same version of Sphinx on Python 3.8 as on the most recent supported versions of Python, because Python 3.14 no longer has `ast.Str` (using `str.Constant` for string literals instead), which causes the oldest version of `sphinx` that runs on Python 3.14 to be `sphinx` 7.2.0, while the newest version that is installable on Python 3.8 is `sphinx` 7.1.2. The immediately preceding commit changes the requirements for the `doc` extra to specify a newer `sphinx` version for Python 3.9 and later. This can't be done on Python 3.8. Because there can be subtle differences in documentation generated with different `sphinx` versions, and because Python 3.8 has been end-of-life for some time, it is not really worth carrying conditional dependencies for the `sphinx` version in `doc/requirements.txt`. Note that, while it is probably not a very good idea to use GitPython (or anything) on Python 3.8 since it is end-of-life, this change does not stop supporting installing GitPython on that or any other version it has been supporting. Installing and using GitPython remains supported all the way back to Python 3.7 at this time. This only affects the `doc` extra and its requirements. This change is analogous to the change made in #1956, which followed up on the change in #1964 in the same way this change follows up on the change in the immediately preceding commit. --- .github/workflows/pythonpackage.yml | 7 ++++++- doc/requirements.txt | 3 +-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3d2cb9b63..8c84f7580 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -30,6 +30,11 @@ jobs: - os-type: ubuntu python-version: "3.7" os-ver: "22.04" + - build-docs: true # We ensure documentation builds, except on very old interpreters. + - python-version: "3.7" + build-docs: false + - python-version: "3.8" + build-docs: false - experimental: false fail-fast: false @@ -110,7 +115,7 @@ jobs: continue-on-error: false - name: Documentation - if: matrix.python-version != '3.7' + if: matrix.build-docs run: | pip install '.[doc]' make -C doc html diff --git a/doc/requirements.txt b/doc/requirements.txt index cbf34cc69..24472ba39 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,3 @@ -sphinx >= 7.4.7, < 8 ; python_version >= "3.9" -sphinx >= 7.1.2, < 7.2 ; python_version < "3.9" +sphinx >= 7.4.7, < 8 sphinx_rtd_theme sphinx-autodoc-typehints From 4b25af2ae8aba951503c8c7055ffa5085bc18f3f Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Mon, 9 Mar 2026 19:16:00 -0400 Subject: [PATCH 1283/1392] Go back to testing free-threaded interpreters only on GNU/Linux This effectively reverts d1ab2e4. It doesn't look like any problems arose, and contrary to my guess, the additional jobs do actually make the checks that we intend to be blocking for PRs take longer, even after all non-macOS checks have completed. --- .github/workflows/pythonpackage.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 8c84f7580..874e18a8f 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -17,6 +17,10 @@ jobs: exclude: - os-type: macos python-version: "3.7" # Not available for the ARM-based macOS runners. + - os-type: macos + python-version: "3.13t" + - os-type: macos + python-version: "3.14t" - os-type: windows python-version: "3.13" # FIXME: Fix and enable Python 3.13 and 3.14 on Windows (#1955). - os-type: windows From 77b1135ef9156a389d83b0d39ad0e1614350c002 Mon Sep 17 00:00:00 2001 From: "GPT 5.4" Date: Fri, 20 Mar 2026 22:12:49 +0000 Subject: [PATCH 1284/1392] fix: resolve active_branch correctly for reftable refs reviewed-by: Sebastian Thiel --- git/repo/base.py | 12 ++++++++++-- test/test_repo.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index 1f543cc57..16807b9fa 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -1042,11 +1042,19 @@ def active_branch(self) -> Head: :raise TypeError: If HEAD is detached. + :raise ValueError: + If HEAD points to the ``.invalid`` ref Git uses to mark refs as + incompatible with older clients. + :return: :class:`~git.refs.head.Head` to the active branch """ - # reveal_type(self.head.reference) # => Reference - return self.head.reference + active_branch = self.head.reference + if active_branch.name == ".invalid": + raise ValueError( + "HEAD points to 'refs/heads/.invalid', which Git uses to mark refs as incompatible with older clients" + ) + return active_branch def blame_incremental(self, rev: str | HEAD | None, file: str, **kwargs: Any) -> Iterator["BlameEntry"]: """Iterator for blame information for the given file at the given revision. diff --git a/test/test_repo.py b/test/test_repo.py index 2a92c2523..544b5c561 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -962,6 +962,46 @@ def test_empty_repo(self, rw_dir): assert "BAD MESSAGE" not in contents, "log is corrupt" + @with_rw_directory + def test_active_branch_raises_value_error_when_head_ref_is_invalid(self, rw_dir): + repo = Repo.init(rw_dir) + with open(osp.join(rw_dir, ".git", "HEAD"), "w") as f: + f.write("ref: refs/heads/.invalid\n") + + self.assertRaisesRegex( + ValueError, + r"refs/heads/\.invalid.*older clients", + lambda: repo.active_branch, + ) + + @with_rw_directory + def test_empty_repo_reftable_active_branch(self, rw_dir): + git = Git(rw_dir) + try: + git.init(ref_format="reftable") + except GitCommandError as err: + if err.status == 129: + pytest.skip("git init --ref-format is not supported by this git version") + raise + + repo = Repo(rw_dir) + self.assertEqual(repo.head.reference.name, ".invalid") + self.assertRaisesRegex( + ValueError, + r"refs/heads/\.invalid.*older clients", + lambda: repo.active_branch, + ) + + @with_rw_directory + def test_active_branch_raises_type_error_when_head_is_detached(self, rw_dir): + repo = Repo.init(rw_dir) + with open(osp.join(rw_dir, "a.txt"), "w") as f: + f.write("a") + repo.index.add(["a.txt"]) + repo.index.commit("initial commit") + repo.git.checkout(repo.head.commit.hexsha) + self.assertRaisesRegex(TypeError, "detached symbolic reference", lambda: repo.active_branch) + def test_merge_base(self): repo = self.rorepo c1 = "f6aa8d1" From 859ea95b6d6207dab3d406f28234e6507ff1f527 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Sun, 22 Mar 2026 00:11:12 -0700 Subject: [PATCH 1285/1392] docs: warn about GitDB performance with large commits Add a warning note in the Object Database section of the tutorial about GitDB failing or becoming extremely slow when traversing trees in repositories with very large commits (thousands of changed files). Directs users to switch to GitCmdObjectDB instead. Closes #2065 --- doc/source/tutorial.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index fd3b14c57..d095d3be3 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -513,6 +513,12 @@ The GitDB is a pure-python implementation of the git object database. It is the repo = Repo("path/to/repo", odbt=GitDB) +.. warning:: + ``GitDB`` may fail or become extremely slow when traversing trees in + repositories with very large commits (thousands of changed files in a + single commit). If you encounter ``RecursionError`` or excessive + slowness during tree traversal, switch to ``GitCmdObjectDB`` instead. + GitCmdObjectDB ============== From 7c5fbc6a95c012e5e70625b78a2e7097c0659fa5 Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya Balusu Date: Tue, 24 Mar 2026 22:36:11 -0400 Subject: [PATCH 1286/1392] Add trailer support for commit creation Add a `trailers` parameter to `Commit.create_from_tree()` and `IndexFile.commit()` that allows appending trailer key-value pairs (e.g. Signed-off-by, Issue) to the commit message at creation time. Trailers can be passed as either a dict or a list of (key, value) tuples, the latter being useful when duplicate keys are needed. The implementation uses `git interpret-trailers` for proper formatting, consistent with the existing trailer parsing in `Commit.trailers_list`. Closes #1998 --- git/index/base.py | 2 ++ git/objects/commit.py | 30 ++++++++++++++++++ test/test_commit.py | 74 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+) diff --git a/git/index/base.py b/git/index/base.py index 93de7933c..2276343f2 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1133,6 +1133,7 @@ def commit( author_date: Union[datetime.datetime, str, None] = None, commit_date: Union[datetime.datetime, str, None] = None, skip_hooks: bool = False, + trailers: Union[None, "Dict[str, str]", "List[Tuple[str, str]]"] = None, ) -> Commit: """Commit the current default index file, creating a :class:`~git.objects.commit.Commit` object. @@ -1169,6 +1170,7 @@ def commit( committer=committer, author_date=author_date, commit_date=commit_date, + trailers=trailers, ) if not skip_hooks: run_commit_hook("post-commit", self) diff --git a/git/objects/commit.py b/git/objects/commit.py index 8c51254a2..3438239b0 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -570,6 +570,7 @@ def create_from_tree( committer: Union[None, Actor] = None, author_date: Union[None, str, datetime.datetime] = None, commit_date: Union[None, str, datetime.datetime] = None, + trailers: Union[None, Dict[str, str], List[Tuple[str, str]]] = None, ) -> "Commit": """Commit the given tree, creating a :class:`Commit` object. @@ -609,6 +610,14 @@ def create_from_tree( :param commit_date: The timestamp for the committer field. + :param trailers: + Optional trailer key-value pairs to append to the commit message. + Can be a dictionary mapping trailer keys to values, or a list of + ``(key, value)`` tuples (useful when the same key appears multiple + times, e.g. multiple ``Signed-off-by`` trailers). Trailers are + appended using ``git interpret-trailers``. + See :manpage:`git-interpret-trailers(1)`. + :return: :class:`Commit` object representing the new commit. @@ -678,6 +687,27 @@ def create_from_tree( tree = repo.tree(tree) # END tree conversion + # APPLY TRAILERS + if trailers: + trailer_args: List[str] = [] + if isinstance(trailers, dict): + for key, val in trailers.items(): + trailer_args.append("--trailer") + trailer_args.append(f"{key}: {val}") + else: + for key, val in trailers: + trailer_args.append("--trailer") + trailer_args.append(f"{key}: {val}") + + cmd = ["git", "interpret-trailers"] + trailer_args + proc: Git.AutoInterrupt = repo.git.execute( # type: ignore[call-overload] + cmd, + as_process=True, + istream=PIPE, + ) + message = proc.communicate(str(message).encode())[0].decode("utf8") + # END apply trailers + # CREATE NEW COMMIT new_commit = cls( repo, diff --git a/test/test_commit.py b/test/test_commit.py index 37c66e3e7..11308cbdb 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -566,3 +566,77 @@ def test_commit_co_authors(self): Actor("test_user_2", "another_user-email@github.com"), Actor("test_user_3", "test_user_3@github.com"), ] + + @with_rw_directory + def test_create_from_tree_with_trailers_dict(self, rw_dir): + """Test that create_from_tree supports adding trailers via a dict.""" + rw_repo = Repo.init(osp.join(rw_dir, "test_trailers_dict")) + path = osp.join(str(rw_repo.working_tree_dir), "hello.txt") + touch(path) + rw_repo.index.add([path]) + tree = rw_repo.index.write_tree() + + trailers = {"Issue": "123", "Signed-off-by": "Test User "} + commit = Commit.create_from_tree( + rw_repo, + tree, + "Test commit with trailers", + head=True, + trailers=trailers, + ) + + assert "Issue: 123" in commit.message + assert "Signed-off-by: Test User " in commit.message + assert commit.trailers_dict == { + "Issue": ["123"], + "Signed-off-by": ["Test User "], + } + + @with_rw_directory + def test_create_from_tree_with_trailers_list(self, rw_dir): + """Test that create_from_tree supports adding trailers via a list of tuples.""" + rw_repo = Repo.init(osp.join(rw_dir, "test_trailers_list")) + path = osp.join(str(rw_repo.working_tree_dir), "hello.txt") + touch(path) + rw_repo.index.add([path]) + tree = rw_repo.index.write_tree() + + trailers = [ + ("Signed-off-by", "Alice "), + ("Signed-off-by", "Bob "), + ("Issue", "456"), + ] + commit = Commit.create_from_tree( + rw_repo, + tree, + "Test commit with multiple trailers", + head=True, + trailers=trailers, + ) + + assert "Signed-off-by: Alice " in commit.message + assert "Signed-off-by: Bob " in commit.message + assert "Issue: 456" in commit.message + assert commit.trailers_dict == { + "Signed-off-by": ["Alice ", "Bob "], + "Issue": ["456"], + } + + @with_rw_directory + def test_index_commit_with_trailers(self, rw_dir): + """Test that IndexFile.commit() supports adding trailers.""" + rw_repo = Repo.init(osp.join(rw_dir, "test_index_trailers")) + path = osp.join(str(rw_repo.working_tree_dir), "hello.txt") + touch(path) + rw_repo.index.add([path]) + + trailers = {"Reviewed-by": "Reviewer "} + commit = rw_repo.index.commit( + "Test index commit with trailers", + trailers=trailers, + ) + + assert "Reviewed-by: Reviewer " in commit.message + assert commit.trailers_dict == { + "Reviewed-by": ["Reviewer "], + } From 9863f501ef5c6aef9b60acc0b490d5cc675aef4e Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke Date: Wed, 25 Mar 2026 16:03:50 +0100 Subject: [PATCH 1287/1392] cmd: fix kwarg formatting in docstring example Update the example to accurately reflect the output of `transform_kwarg`. When a key is longer than one letter and its value is a non-empty, non-boolean type, it is transformed into the `--key=value` format, rather than missing the double dashes or using spaces. Signed-off-by: Uwe Schwaeke --- git/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index 78a9f4c78..b529bcc10 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1572,7 +1572,7 @@ def _call_process( turns into:: - git rev-list max-count 10 --header master + git rev-list --max-count=10 --header=master :return: Same as :meth:`execute`. If no args are given, used :meth:`execute`'s From 0391926ac58b926ecdefc54a5e475555f494b8f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Apr 2026 17:17:25 +0000 Subject: [PATCH 1288/1392] Bump https://github.com/astral-sh/ruff-pre-commit Bumps the pre-commit group with 1 update: [https://github.com/astral-sh/ruff-pre-commit](https://github.com/astral-sh/ruff-pre-commit). Updates `https://github.com/astral-sh/ruff-pre-commit` from v0.15.5 to 0.15.8 - [Release notes](https://github.com/astral-sh/ruff-pre-commit/releases) - [Commits](https://github.com/astral-sh/ruff-pre-commit/compare/v0.15.5...v0.15.8) --- updated-dependencies: - dependency-name: https://github.com/astral-sh/ruff-pre-commit dependency-version: 0.15.8 dependency-type: direct:production dependency-group: pre-commit ... Signed-off-by: dependabot[bot] --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3bd9cbce9..617111e1d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: exclude: ^test/fixtures/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.5 + rev: v0.15.8 hooks: - id: ruff-check args: ["--fix"] From af0933cadc14f0df7a3463b655793d59cd586c46 Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya Balusu Date: Mon, 6 Apr 2026 08:56:20 -0700 Subject: [PATCH 1289/1392] Use configured git executable and finalize process for trailer creation --- git/objects/commit.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 3438239b0..6ea252395 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -699,13 +699,15 @@ def create_from_tree( trailer_args.append("--trailer") trailer_args.append(f"{key}: {val}") - cmd = ["git", "interpret-trailers"] + trailer_args + cmd = [repo.git.GIT_PYTHON_GIT_EXECUTABLE, "interpret-trailers"] + trailer_args proc: Git.AutoInterrupt = repo.git.execute( # type: ignore[call-overload] cmd, as_process=True, istream=PIPE, ) - message = proc.communicate(str(message).encode())[0].decode("utf8") + stdout_bytes, _ = proc.communicate(str(message).encode()) + finalize_process(proc) + message = stdout_bytes.decode("utf8") # END apply trailers # CREATE NEW COMMIT From bd58716966ffb231f96aeada76c5159d5b4f9beb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 12 Apr 2026 22:46:53 +0000 Subject: [PATCH 1290/1392] Use consistent interpret-trailers encoding Agent-Logs-Url: https://github.com/gitpython-developers/GitPython/sessions/1a855cb6-0111-4f52-b48d-46417aec5bde Co-authored-by: Byron <63622+Byron@users.noreply.github.com> --- git/objects/commit.py | 31 ++++++++++++++----------------- test/test_commit.py | 25 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 6ea252395..081ccf402 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -450,14 +450,7 @@ def trailers_list(self) -> List[Tuple[str, str]]: :return: List containing key-value tuples of whitespace stripped trailer information. """ - cmd = ["git", "interpret-trailers", "--parse"] - proc: Git.AutoInterrupt = self.repo.git.execute( # type: ignore[call-overload] - cmd, - as_process=True, - istream=PIPE, - ) - trailer: str = proc.communicate(str(self.message).encode())[0].decode("utf8") - trailer = trailer.strip() + trailer = self._interpret_trailers(self.repo, self.message, ["--parse"], self.encoding).strip() if not trailer: return [] @@ -469,6 +462,18 @@ def trailers_list(self) -> List[Tuple[str, str]]: return trailer_list + @staticmethod + def _interpret_trailers(repo: "Repo", message: str, trailer_args: Sequence[str], encoding: str) -> str: + cmd = [repo.git.GIT_PYTHON_GIT_EXECUTABLE, "interpret-trailers", *trailer_args] + proc: Git.AutoInterrupt = repo.git.execute( # type: ignore[call-overload] + cmd, + as_process=True, + istream=PIPE, + ) + stdout_bytes, _ = proc.communicate(message.encode(encoding, errors="strict")) + finalize_process(proc) + return stdout_bytes.decode(encoding, errors="strict") + @property def trailers_dict(self) -> Dict[str, List[str]]: """Get the trailers of the message as a dictionary. @@ -699,15 +704,7 @@ def create_from_tree( trailer_args.append("--trailer") trailer_args.append(f"{key}: {val}") - cmd = [repo.git.GIT_PYTHON_GIT_EXECUTABLE, "interpret-trailers"] + trailer_args - proc: Git.AutoInterrupt = repo.git.execute( # type: ignore[call-overload] - cmd, - as_process=True, - istream=PIPE, - ) - stdout_bytes, _ = proc.communicate(str(message).encode()) - finalize_process(proc) - message = stdout_bytes.decode("utf8") + message = cls._interpret_trailers(repo, str(message), trailer_args, conf_encoding) # END apply trailers # CREATE NEW COMMIT diff --git a/test/test_commit.py b/test/test_commit.py index 11308cbdb..5ea6642c0 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -622,6 +622,31 @@ def test_create_from_tree_with_trailers_list(self, rw_dir): "Issue": ["456"], } + @with_rw_directory + def test_create_from_tree_with_non_utf8_trailers(self, rw_dir): + """Test that trailer creation and parsing respect the configured commit encoding.""" + rw_repo = Repo.init(osp.join(rw_dir, "test_trailers_non_utf8")) + with rw_repo.config_writer() as writer: + writer.set_value("i18n", "commitencoding", "ISO-8859-1") + + path = osp.join(str(rw_repo.working_tree_dir), "hello.txt") + touch(path) + rw_repo.index.add([path]) + tree = rw_repo.index.write_tree() + + commit = Commit.create_from_tree( + rw_repo, + tree, + "Résumé", + head=True, + trailers={"Reviewed-by": "André "}, + ) + + assert commit.encoding == "ISO-8859-1" + assert "Résumé" in commit.message + assert "Reviewed-by: André " in commit.message + assert commit.trailers_list == [("Reviewed-by", "André ")] + @with_rw_directory def test_index_commit_with_trailers(self, rw_dir): """Test that IndexFile.commit() supports adding trailers.""" From 7cdf9c7fb5d27dfee1e22fa81fc28d9e538d58a7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 12 Apr 2026 22:47:59 +0000 Subject: [PATCH 1291/1392] Normalize interpret-trailers subprocess IO Agent-Logs-Url: https://github.com/gitpython-developers/GitPython/sessions/1a855cb6-0111-4f52-b48d-46417aec5bde Co-authored-by: Byron <63622+Byron@users.noreply.github.com> --- git/objects/commit.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 081ccf402..a8bb5e852 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -450,7 +450,7 @@ def trailers_list(self) -> List[Tuple[str, str]]: :return: List containing key-value tuples of whitespace stripped trailer information. """ - trailer = self._interpret_trailers(self.repo, self.message, ["--parse"], self.encoding).strip() + trailer = self._interpret_trailers(self.repo, self.message, ["--parse"]).strip() if not trailer: return [] @@ -462,17 +462,17 @@ def trailers_list(self) -> List[Tuple[str, str]]: return trailer_list - @staticmethod - def _interpret_trailers(repo: "Repo", message: str, trailer_args: Sequence[str], encoding: str) -> str: + @classmethod + def _interpret_trailers(cls, repo: "Repo", message: str, trailer_args: Sequence[str]) -> str: cmd = [repo.git.GIT_PYTHON_GIT_EXECUTABLE, "interpret-trailers", *trailer_args] proc: Git.AutoInterrupt = repo.git.execute( # type: ignore[call-overload] cmd, as_process=True, istream=PIPE, ) - stdout_bytes, _ = proc.communicate(message.encode(encoding, errors="strict")) + stdout_bytes, _ = proc.communicate(message.encode(cls.default_encoding, errors="strict")) finalize_process(proc) - return stdout_bytes.decode(encoding, errors="strict") + return stdout_bytes.decode(cls.default_encoding, errors="strict") @property def trailers_dict(self) -> Dict[str, List[str]]: @@ -704,7 +704,7 @@ def create_from_tree( trailer_args.append("--trailer") trailer_args.append(f"{key}: {val}") - message = cls._interpret_trailers(repo, str(message), trailer_args, conf_encoding) + message = cls._interpret_trailers(repo, str(message), trailer_args) # END apply trailers # CREATE NEW COMMIT From 1e2a895ef55911b500b28360ee97c37e6678c014 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 12 Apr 2026 23:43:59 +0000 Subject: [PATCH 1292/1392] Handle mypy CI typing failure Agent-Logs-Url: https://github.com/gitpython-developers/GitPython/sessions/3d3e7ffc-d3af-478e-9c6c-128731cdd102 Co-authored-by: Byron <63622+Byron@users.noreply.github.com> --- git/objects/commit.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index a8bb5e852..6e5550e37 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -463,14 +463,15 @@ def trailers_list(self) -> List[Tuple[str, str]]: return trailer_list @classmethod - def _interpret_trailers(cls, repo: "Repo", message: str, trailer_args: Sequence[str]) -> str: + def _interpret_trailers(cls, repo: "Repo", message: Union[str, bytes], trailer_args: Sequence[str]) -> str: cmd = [repo.git.GIT_PYTHON_GIT_EXECUTABLE, "interpret-trailers", *trailer_args] proc: Git.AutoInterrupt = repo.git.execute( # type: ignore[call-overload] cmd, as_process=True, istream=PIPE, ) - stdout_bytes, _ = proc.communicate(message.encode(cls.default_encoding, errors="strict")) + message_bytes = message if isinstance(message, bytes) else message.encode(cls.default_encoding, errors="strict") + stdout_bytes, _ = proc.communicate(message_bytes) finalize_process(proc) return stdout_bytes.decode(cls.default_encoding, errors="strict") From 34ec40dc70ab897127184ba88792596fce78d44b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 03:37:51 +0000 Subject: [PATCH 1293/1392] Use commit encoding for trailer parsing Agent-Logs-Url: https://github.com/gitpython-developers/GitPython/sessions/519084d5-d5e2-4486-a9cc-5c258e596e13 Co-authored-by: Byron <63622+Byron@users.noreply.github.com> --- git/objects/commit.py | 14 ++++++++++---- test/test_commit.py | 24 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 6e5550e37..206b6fcc3 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -450,7 +450,7 @@ def trailers_list(self) -> List[Tuple[str, str]]: :return: List containing key-value tuples of whitespace stripped trailer information. """ - trailer = self._interpret_trailers(self.repo, self.message, ["--parse"]).strip() + trailer = self._interpret_trailers(self.repo, self.message, ["--parse"], encoding=self.encoding).strip() if not trailer: return [] @@ -463,17 +463,23 @@ def trailers_list(self) -> List[Tuple[str, str]]: return trailer_list @classmethod - def _interpret_trailers(cls, repo: "Repo", message: Union[str, bytes], trailer_args: Sequence[str]) -> str: + def _interpret_trailers( + cls, + repo: "Repo", + message: Union[str, bytes], + trailer_args: Sequence[str], + encoding: str = default_encoding, + ) -> str: cmd = [repo.git.GIT_PYTHON_GIT_EXECUTABLE, "interpret-trailers", *trailer_args] proc: Git.AutoInterrupt = repo.git.execute( # type: ignore[call-overload] cmd, as_process=True, istream=PIPE, ) - message_bytes = message if isinstance(message, bytes) else message.encode(cls.default_encoding, errors="strict") + message_bytes = message if isinstance(message, bytes) else message.encode(encoding, errors="strict") stdout_bytes, _ = proc.communicate(message_bytes) finalize_process(proc) - return stdout_bytes.decode(cls.default_encoding, errors="strict") + return stdout_bytes.decode(encoding, errors="strict") @property def trailers_dict(self) -> Dict[str, List[str]]: diff --git a/test/test_commit.py b/test/test_commit.py index 5ea6642c0..b3b5f03ec 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -647,6 +647,30 @@ def test_create_from_tree_with_non_utf8_trailers(self, rw_dir): assert "Reviewed-by: André " in commit.message assert commit.trailers_list == [("Reviewed-by", "André ")] + @with_rw_directory + def test_trailers_list_with_non_utf8_message_bytes(self, rw_dir): + """Test that trailer parsing handles non-UTF-8 commit message bytes.""" + rw_repo = Repo.init(osp.join(rw_dir, "test_trailers_non_utf8_bytes")) + with rw_repo.config_writer() as writer: + writer.set_value("i18n", "commitencoding", "ISO-8859-1") + + path = osp.join(str(rw_repo.working_tree_dir), "hello.txt") + touch(path) + rw_repo.index.add([path]) + tree = rw_repo.index.write_tree() + + commit = Commit.create_from_tree( + rw_repo, + tree, + "Résumé", + head=True, + trailers={"Reviewed-by": "André "}, + ) + + commit.message = commit.message.encode(commit.encoding) + + assert commit.trailers_list == [("Reviewed-by", "André ")] + @with_rw_directory def test_index_commit_with_trailers(self, rw_dir): """Test that IndexFile.commit() supports adding trailers.""" From 4aa8157cbb4d70f18f599c35cb358b500a364b59 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 03:39:22 +0000 Subject: [PATCH 1294/1392] Refine trailer encoding review follow-up Agent-Logs-Url: https://github.com/gitpython-developers/GitPython/sessions/519084d5-d5e2-4486-a9cc-5c258e596e13 Co-authored-by: Byron <63622+Byron@users.noreply.github.com> --- test/test_commit.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/test_commit.py b/test/test_commit.py index b3b5f03ec..8982db1e3 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -667,9 +667,14 @@ def test_trailers_list_with_non_utf8_message_bytes(self, rw_dir): trailers={"Reviewed-by": "André "}, ) - commit.message = commit.message.encode(commit.encoding) + bytes_commit = Commit( + rw_repo, + commit.binsha, + message=commit.message.encode(commit.encoding), + encoding=commit.encoding, + ) - assert commit.trailers_list == [("Reviewed-by", "André ")] + assert bytes_commit.trailers_list == [("Reviewed-by", "André ")] @with_rw_directory def test_index_commit_with_trailers(self, rw_dir): From 633abdbcd2e897656c998289e48369080a05f600 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 04:23:04 +0000 Subject: [PATCH 1295/1392] Fix trailer subprocess lifetime Agent-Logs-Url: https://github.com/gitpython-developers/GitPython/sessions/3cc0bd6d-d54d-4299-9a18-1576c2a91c12 Co-authored-by: Byron <63622+Byron@users.noreply.github.com> --- git/objects/commit.py | 10 ++++++---- test/test_commit.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 206b6fcc3..da7677ee0 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -470,16 +470,18 @@ def _interpret_trailers( trailer_args: Sequence[str], encoding: str = default_encoding, ) -> str: + message_bytes = message if isinstance(message, bytes) else message.encode(encoding, errors="strict") cmd = [repo.git.GIT_PYTHON_GIT_EXECUTABLE, "interpret-trailers", *trailer_args] proc: Git.AutoInterrupt = repo.git.execute( # type: ignore[call-overload] cmd, as_process=True, istream=PIPE, ) - message_bytes = message if isinstance(message, bytes) else message.encode(encoding, errors="strict") - stdout_bytes, _ = proc.communicate(message_bytes) - finalize_process(proc) - return stdout_bytes.decode(encoding, errors="strict") + try: + stdout_bytes, _ = proc.communicate(message_bytes) + return stdout_bytes.decode(encoding, errors="strict") + finally: + finalize_process(proc) @property def trailers_dict(self) -> Dict[str, List[str]]: diff --git a/test/test_commit.py b/test/test_commit.py index 8982db1e3..b56ad3a18 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -676,6 +676,17 @@ def test_trailers_list_with_non_utf8_message_bytes(self, rw_dir): assert bytes_commit.trailers_list == [("Reviewed-by", "André ")] + def test_interpret_trailers_encodes_before_launching_process(self): + """Test that encoding failures happen before spawning interpret-trailers.""" + repo = Mock() + repo.git = Mock() + repo.git.GIT_PYTHON_GIT_EXECUTABLE = "git" + + with self.assertRaises(UnicodeEncodeError): + Commit._interpret_trailers(repo, "Euro: €", ["--parse"], encoding="ISO-8859-1") + + repo.git.execute.assert_not_called() + @with_rw_directory def test_index_commit_with_trailers(self, rw_dir): """Test that IndexFile.commit() supports adding trailers.""" From d966a0deabe3c8cf09ba3d1b0f54a29bdbdb4f1d Mon Sep 17 00:00:00 2001 From: Enji Cooper Date: Fri, 17 Apr 2026 01:00:40 -0700 Subject: [PATCH 1296/1392] git.cmd.Git.execute(..): fix `with_stdout=False` In the event the end-user called one of the APIs with `with_stdout=False`, i.e., they didn't want to capture stdout, the code would crash with an AttributeError or ValueError when trying to dereference the stdout/stderr streams attached to `Popen(..)` objects. Be more defensive by checking the streams first to make sure they're not `None` before trying to access their corresponding attributes. Add myself to AUTHORS and add corresponding regression tests for the change. Signed-off-by: Enji Cooper --- AUTHORS | 1 + git/cmd.py | 20 ++++++++++++-------- test/test_git.py | 20 ++++++++++++++++++++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/AUTHORS b/AUTHORS index b57113edd..15333e1e5 100644 --- a/AUTHORS +++ b/AUTHORS @@ -56,5 +56,6 @@ Contributors are: -Ethan Lin -Jonas Scharpf -Gordon Marx +-Enji Cooper Portions derived from other open source works and are clearly marked. diff --git a/git/cmd.py b/git/cmd.py index b529bcc10..d5fbc7736 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1364,25 +1364,29 @@ def communicate() -> Tuple[AnyStr, AnyStr]: if output_stream is None: stdout_value, stderr_value = communicate() # Strip trailing "\n". - if stdout_value.endswith(newline) and strip_newline_in_stdout: # type: ignore[arg-type] + if stdout_value is not None and stdout_value.endswith(newline) and strip_newline_in_stdout: # type: ignore[arg-type] stdout_value = stdout_value[:-1] - if stderr_value.endswith(newline): # type: ignore[arg-type] + if stderr_value is not None and stderr_value.endswith(newline): # type: ignore[arg-type] stderr_value = stderr_value[:-1] status = proc.returncode else: max_chunk_size = max_chunk_size if max_chunk_size and max_chunk_size > 0 else io.DEFAULT_BUFFER_SIZE - stream_copy(proc.stdout, output_stream, max_chunk_size) - stdout_value = proc.stdout.read() - stderr_value = proc.stderr.read() + if proc.stdout is not None: + stream_copy(proc.stdout, output_stream, max_chunk_size) + stdout_value = proc.stdout.read() + if proc.stderr is not None: + stderr_value = proc.stderr.read() # Strip trailing "\n". - if stderr_value.endswith(newline): # type: ignore[arg-type] + if stderr_value is not None and stderr_value.endswith(newline): # type: ignore[arg-type] stderr_value = stderr_value[:-1] status = proc.wait() # END stdout handling finally: - proc.stdout.close() - proc.stderr.close() + if proc.stdout is not None: + proc.stdout.close() + if proc.stderr is not None: + proc.stderr.close() if self.GIT_PYTHON_TRACE == "full": cmdstr = " ".join(redacted_command) diff --git a/test/test_git.py b/test/test_git.py index 4a54d0d9b..da50fdfe8 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -6,6 +6,7 @@ import contextlib import gc import inspect +import io import logging import os import os.path as osp @@ -201,6 +202,25 @@ def test_it_logs_istream_summary_for_stdin(self, case): def test_it_executes_git_and_returns_result(self): self.assertRegex(self.git.execute(["git", "version"]), r"^git version [\d\.]{2}.*$") + def test_it_output_stream_with_stdout_is_false(self): + temp_stream = io.BytesIO() + self.git.execute( + ["git", "version"], + output_stream=temp_stream, + with_stdout=False, + ) + self.assertEqual(temp_stream.tell(), 0) + + def test_it_executes_git_without_stdout_redirect(self): + returncode, stdout, stderr = self.git.execute( + ["git", "version"], + with_extended_output=True, + with_stdout=False, + ) + self.assertEqual(returncode, 0) + self.assertIsNone(stdout) + self.assertIsNotNone(stderr) + @ddt.data( # chdir_to_repo, shell, command, use_shell_impostor (False, False, ["git", "version"], False), From 6fc474265d863cbb9fbabdbfcc957f27cea2b5c4 Mon Sep 17 00:00:00 2001 From: Enji Cooper Date: Fri, 17 Apr 2026 09:40:43 -0700 Subject: [PATCH 1297/1392] test_avoids_changing...: don't leave test artifacts behind Prior to this the test would fail [silently] on my macOS host during the test and then pytest would complain loudly about it being an issue post-session (regardless of whether or not the test was being run). Squash the unwritable directory to mute noise complaints from pytest. Signed-off-by: Enji Cooper --- test/test_util.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index 000830f41..e7453769a 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -113,7 +113,7 @@ def test_deletes_dir_with_readonly_files(self, tmp_path): sys.platform == "cygwin", reason="Cygwin can't set the permissions that make the test meaningful.", ) - def test_avoids_changing_permissions_outside_tree(self, tmp_path): + def test_avoids_changing_permissions_outside_tree(self, tmp_path, request): # Automatically works on Windows, but on Unix requires either special handling # or refraining from attempting to fix PermissionError by making chmod calls. @@ -125,9 +125,32 @@ def test_avoids_changing_permissions_outside_tree(self, tmp_path): dir2 = tmp_path / "dir2" dir2.mkdir() - (dir2 / "symlink").symlink_to(dir1 / "file") + symlink = dir2 / "symlink" + symlink.symlink_to(dir1 / "file") dir2.chmod(stat.S_IRUSR | stat.S_IXUSR) + def preen_dir2(): + """Don't leave unwritable directories behind. + + pytest has difficulties cleaning up after the fact on some platforms, + e.g., macOS, and whines incessantly until the issue is resolved--regardless + of the pytest session. + """ + rwx = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + if not dir2.exists(): + return + if symlink.exists(): + try: + # Try lchmod first, if the platform supports it. + symlink.lchmod(rwx) + except NotImplementedError: + # The platform (probably win32) doesn't support lchmod; fall back to chmod. + symlink.chmod(rwx) + dir2.chmod(rwx) + rmtree(dir2) + + request.addfinalizer(preen_dir2) + try: rmtree(dir2) except PermissionError: From c9a26789d88b18f8b4620f37307df2976292d2a0 Mon Sep 17 00:00:00 2001 From: "GPT 5.4" Date: Tue, 21 Apr 2026 09:30:29 +0800 Subject: [PATCH 1298/1392] Make sure that multi-options are checked after splitting them with `shlex` Co-authored-by: Sebastian Thiel --- git/repo/base.py | 4 ++-- test/test_clone.py | 18 ++++++++++++++++++ test/test_submodule.py | 11 +++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index 16807b9fa..96c78df56 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -1386,8 +1386,8 @@ def _clone( Git.check_unsafe_protocols(url) if not allow_unsafe_options: Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=cls.unsafe_git_clone_options) - if not allow_unsafe_options and multi_options: - Git.check_unsafe_options(options=multi_options, unsafe_options=cls.unsafe_git_clone_options) + if not allow_unsafe_options and multi: + Git.check_unsafe_options(options=multi, unsafe_options=cls.unsafe_git_clone_options) proc = git.clone( multi, diff --git a/test/test_clone.py b/test/test_clone.py index 143a3b51f..768efbba6 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -137,6 +137,15 @@ def test_clone_unsafe_options(self, rw_repo): rw_repo.clone(tmp_dir, **unsafe_option) assert not tmp_file.exists() + @with_rw_repo("HEAD") + def test_clone_unsafe_options_are_checked_after_splitting_multi_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + payload = "--single-branch --config protocol.ext.allow=always" + + with self.assertRaises(UnsafeOptionError): + rw_repo.clone(tmp_dir, multi_options=[payload]) + @pytest.mark.xfail( sys.platform == "win32", reason=( @@ -216,6 +225,15 @@ def test_clone_from_unsafe_options(self, rw_repo): Repo.clone_from(rw_repo.working_dir, tmp_dir, **unsafe_option) assert not tmp_file.exists() + @with_rw_repo("HEAD") + def test_clone_from_unsafe_options_are_checked_after_splitting_multi_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + payload = "--single-branch --config protocol.ext.allow=always" + + with self.assertRaises(UnsafeOptionError): + Repo.clone_from(rw_repo.working_dir, tmp_dir, multi_options=[payload]) + @pytest.mark.xfail( sys.platform == "win32", reason=( diff --git a/test/test_submodule.py b/test/test_submodule.py index 47647f2a1..63bb007de 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -1332,6 +1332,17 @@ def test_submodule_update_unsafe_options(self, rw_repo): submodule.update(clone_multi_options=[unsafe_option]) assert not tmp_file.exists() + @with_rw_repo("HEAD") + def test_submodule_update_unsafe_options_are_checked_after_splitting_multi_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = Path(tdir) + payload = "--single-branch --config protocol.ext.allow=always" + submodule = Submodule(rw_repo, b"\0" * 20, name="new", path="new", url=str(tmp_dir)) + + with self.assertRaises(UnsafeOptionError): + submodule.update(clone_multi_options=[payload]) + assert not submodule.module_exists() + @with_rw_repo("HEAD") def test_submodule_update_unsafe_options_allowed(self, rw_repo): with tempfile.TemporaryDirectory() as tdir: From 142195888e713542189533a52cdfc333f05c3af6 Mon Sep 17 00:00:00 2001 From: w Date: Mon, 20 Apr 2026 23:29:50 -0400 Subject: [PATCH 1299/1392] Block unsafe underscored git kwargs / Fix for GHSA-rpm5-65cw-6hj4 --- git/cmd.py | 21 +++++++++++++-------- test/test_clone.py | 2 ++ test/test_git.py | 16 ++++++++++++++++ test/test_remote.py | 5 +++-- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index d5fbc7736..3a4b69572 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -944,6 +944,12 @@ def check_unsafe_protocols(cls, url: str) -> None: f"The `{protocol}::` protocol looks suspicious, use `allow_unsafe_protocols=True` to allow it." ) + @classmethod + def _canonicalize_option_name(cls, option: str) -> str: + """Normalize an option or kwarg name for unsafe-option checks.""" + option_name = option.lstrip("-").split("=", 1)[0].split(None, 1)[0] + return dashify(option_name) + @classmethod def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> None: """Check for unsafe options. @@ -951,15 +957,14 @@ def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> Some options that are passed to ``git `` can be used to execute arbitrary commands. These are blocked by default. """ - # Options can be of the form `foo`, `--foo bar`, or `--foo=bar`, so we need to - # check if they start with "--foo" or if they are equal to "foo". - bare_unsafe_options = [option.lstrip("-") for option in unsafe_options] + # Options can be of the form `foo`, `--foo`, `--foo bar`, or `--foo=bar`. + canonical_unsafe_options = {cls._canonicalize_option_name(option): option for option in unsafe_options} for option in options: - for unsafe_option, bare_option in zip(unsafe_options, bare_unsafe_options): - if option.startswith(unsafe_option) or option == bare_option: - raise UnsafeOptionError( - f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it." - ) + unsafe_option = canonical_unsafe_options.get(cls._canonicalize_option_name(option)) + if unsafe_option is not None: + raise UnsafeOptionError( + f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it." + ) AutoInterrupt: TypeAlias = _AutoInterrupt diff --git a/test/test_clone.py b/test/test_clone.py index 768efbba6..653d50aa3 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -128,6 +128,7 @@ def test_clone_unsafe_options(self, rw_repo): unsafe_options = [ {"upload-pack": f"touch {tmp_file}"}, + {"upload_pack": f"touch {tmp_file}"}, {"u": f"touch {tmp_file}"}, {"config": "protocol.ext.allow=always"}, {"c": "protocol.ext.allow=always"}, @@ -216,6 +217,7 @@ def test_clone_from_unsafe_options(self, rw_repo): unsafe_options = [ {"upload-pack": f"touch {tmp_file}"}, + {"upload_pack": f"touch {tmp_file}"}, {"u": f"touch {tmp_file}"}, {"config": "protocol.ext.allow=always"}, {"c": "protocol.ext.allow=always"}, diff --git a/test/test_git.py b/test/test_git.py index da50fdfe8..24b60af9d 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -27,6 +27,7 @@ import ddt from git import Git, GitCommandError, GitCommandNotFound, Repo, cmd, refresh +from git.exc import UnsafeOptionError from git.util import cwd, finalize_process from test.lib import TestBase, fixture_path, with_rw_directory @@ -154,6 +155,21 @@ def test_it_transforms_kwargs_into_git_command_arguments(self): res = self.git.transform_kwargs(**{"s": True, "t": True}) self.assertEqual({"-s", "-t"}, set(res)) + def test_check_unsafe_options_normalizes_kwargs(self): + cases = [ + (["upload_pack"], ["--upload-pack"]), + (["receive_pack"], ["--receive-pack"]), + (["exec"], ["--exec"]), + (["u"], ["-u"]), + (["c"], ["-c"]), + (["--upload-pack=/tmp/helper"], ["--upload-pack"]), + (["--config core.filemode=false"], ["--config"]), + ] + + for options, unsafe_options in cases: + with self.assertRaises(UnsafeOptionError): + Git.check_unsafe_options(options=options, unsafe_options=unsafe_options) + _shell_cases = ( # value_in_call, value_from_class, expected_popen_arg (None, False, False), diff --git a/test/test_remote.py b/test/test_remote.py index b1d686f05..0551060cf 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -827,7 +827,7 @@ def test_fetch_unsafe_options(self, rw_repo): remote = rw_repo.remote("origin") tmp_dir = Path(tdir) tmp_file = tmp_dir / "pwn" - unsafe_options = [{"upload-pack": f"touch {tmp_file}"}] + unsafe_options = [{"upload-pack": f"touch {tmp_file}"}, {"upload_pack": f"touch {tmp_file}"}] for unsafe_option in unsafe_options: with self.assertRaises(UnsafeOptionError): remote.fetch(**unsafe_option) @@ -895,7 +895,7 @@ def test_pull_unsafe_options(self, rw_repo): remote = rw_repo.remote("origin") tmp_dir = Path(tdir) tmp_file = tmp_dir / "pwn" - unsafe_options = [{"upload-pack": f"touch {tmp_file}"}] + unsafe_options = [{"upload-pack": f"touch {tmp_file}"}, {"upload_pack": f"touch {tmp_file}"}] for unsafe_option in unsafe_options: with self.assertRaises(UnsafeOptionError): remote.pull(**unsafe_option) @@ -966,6 +966,7 @@ def test_push_unsafe_options(self, rw_repo): unsafe_options = [ { "receive-pack": f"touch {tmp_file}", + "receive_pack": f"touch {tmp_file}", "exec": f"touch {tmp_file}", } ] From 9aed7cf8c20f69effcfcf7ebef09f312f73ab826 Mon Sep 17 00:00:00 2001 From: w Date: Mon, 20 Apr 2026 23:43:59 -0400 Subject: [PATCH 1300/1392] linter fix --- git/cmd.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 3a4b69572..02d56616c 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -962,9 +962,7 @@ def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> for option in options: unsafe_option = canonical_unsafe_options.get(cls._canonicalize_option_name(option)) if unsafe_option is not None: - raise UnsafeOptionError( - f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it." - ) + raise UnsafeOptionError(f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it.") AutoInterrupt: TypeAlias = _AutoInterrupt From 43d92dec4683568d11495956dd556161f17c3ea8 Mon Sep 17 00:00:00 2001 From: w Date: Tue, 21 Apr 2026 12:03:20 -0400 Subject: [PATCH 1301/1392] git.cmd: harden unsafe option canonicalization and isolate push test cases --- git/cmd.py | 15 ++++++++++++--- test/test_remote.py | 15 ++++++--------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 02d56616c..096900819 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -946,9 +946,18 @@ def check_unsafe_protocols(cls, url: str) -> None: @classmethod def _canonicalize_option_name(cls, option: str) -> str: - """Normalize an option or kwarg name for unsafe-option checks.""" - option_name = option.lstrip("-").split("=", 1)[0].split(None, 1)[0] - return dashify(option_name) + """Return the option name used for unsafe-option checks. + + Examples: + ``"--upload-pack=/tmp/helper"`` -> ``"upload-pack"`` + ``"upload_pack"`` -> ``"upload-pack"`` + ``"--config core.filemode=false"`` -> ``"config"`` + """ + option_name = option.lstrip("-").split("=", 1)[0] + option_tokens = option_name.split(None, 1) + if not option_tokens: + return "" + return dashify(option_tokens[0]) @classmethod def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> None: diff --git a/test/test_remote.py b/test/test_remote.py index 0551060cf..1c627127a 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -964,11 +964,9 @@ def test_push_unsafe_options(self, rw_repo): tmp_dir = Path(tdir) tmp_file = tmp_dir / "pwn" unsafe_options = [ - { - "receive-pack": f"touch {tmp_file}", - "receive_pack": f"touch {tmp_file}", - "exec": f"touch {tmp_file}", - } + {"receive-pack": f"touch {tmp_file}"}, + {"receive_pack": f"touch {tmp_file}"}, + {"exec": f"touch {tmp_file}"}, ] for unsafe_option in unsafe_options: assert not tmp_file.exists() @@ -992,10 +990,9 @@ def test_push_unsafe_options_allowed(self, rw_repo): tmp_dir = Path(tdir) tmp_file = tmp_dir / "pwn" unsafe_options = [ - { - "receive-pack": f"touch {tmp_file}", - "exec": f"touch {tmp_file}", - } + {"receive-pack": f"touch {tmp_file}"}, + {"receive_pack": f"touch {tmp_file}"}, + {"exec": f"touch {tmp_file}"}, ] for unsafe_option in unsafe_options: # The options will be allowed, but the command will fail. From 4199cb89755f705801a4cb241723325b46201f51 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 22 Apr 2026 10:35:03 +0800 Subject: [PATCH 1302/1392] bump version to 3.1.47 --- VERSION | 2 +- doc/source/changes.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fd84d1e83..e1ace7c6e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.46 +3.1.47 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 9b82e7513..90b2e0739 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,15 @@ Changelog ========= +3.1.47 +====== + +Address various security issues related to bypassing injection-protection +of unsafe Git flags. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.47 + 3.1.46 ====== From 4c6ec603e21ce81ea37bb69a17a7aaf82f4c4eb2 Mon Sep 17 00:00:00 2001 From: Menashe Eliezer Date: Mon, 20 Apr 2026 11:59:39 +0200 Subject: [PATCH 1303/1392] fix: support Repo() autodiscovery from linked worktree GIT_DIR Handle linked worktree git directories when GIT_DIR points to .git/worktrees/. Previously Repo() could fail with InvalidGitRepositoryError in this scenario, while Repo(os.getcwd()) worked correctly. Add regression test to cover autodiscovery in linked worktrees. --- git/repo/base.py | 22 ++++++++++++++++++++++ test/test_repo.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/git/repo/base.py b/git/repo/base.py index 16807b9fa..7c3b5b2cc 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -242,6 +242,28 @@ def __init__( # It's important to normalize the paths, as submodules will otherwise # initialize their repo instances with paths that depend on path-portions # that will not exist after being removed. It's just cleaner. + if ( + osp.isfile(osp.join(curpath, "gitdir")) + and osp.isfile(osp.join(curpath, "commondir")) + and osp.isfile(osp.join(curpath, "HEAD")) + ): + git_dir = curpath + + if "GIT_WORK_TREE" in os.environ: + self._working_tree_dir = os.getenv("GIT_WORK_TREE") + else: + # Linked worktree administrative directories store the path to the + # worktree's .git file in their gitdir file (without "gitdir: " prefix). + with open(osp.join(git_dir, "gitdir")) as fp: + worktree_gitfile = fp.read().strip() + + if not osp.isabs(worktree_gitfile): + worktree_gitfile = osp.normpath(osp.join(git_dir, worktree_gitfile)) + + self._working_tree_dir = osp.dirname(worktree_gitfile) + + break + if is_git_dir(curpath): git_dir = curpath # from man git-config : core.worktree diff --git a/test/test_repo.py b/test/test_repo.py index 544b5c561..65d122fd7 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -1126,6 +1126,42 @@ def test_git_work_tree_env(self, rw_dir): self.assertEqual(r.working_tree_dir, repo_dir) self.assertEqual(r.working_dir, repo_dir) + @with_rw_directory + def test_git_work_tree_env_in_linked_worktree(self, rw_dir): + """Check that Repo() autodiscovers a linked worktree when GIT_DIR is set.""" + git = Git(rw_dir) + if git.version_info[:3] < (2, 5, 1): + raise RuntimeError("worktree feature unsupported (test needs git 2.5.1 or later)") + + rw_master = self.rorepo.clone(join_path_native(rw_dir, "master_repo")) + branch = rw_master.create_head("bbbbbbbb") + worktree_path = join_path_native(rw_dir, "worktree_repo") + if Git.is_cygwin(): + worktree_path = cygpath(worktree_path) + + rw_master.git.worktree("add", worktree_path, branch.name) + + git_dir = Git(worktree_path).rev_parse("--git-dir") + + patched_env = dict(os.environ) + patched_env["GIT_DIR"] = git_dir + patched_env.pop("GIT_WORK_TREE", None) + patched_env.pop("GIT_COMMON_DIR", None) + + with mock.patch.dict(os.environ, patched_env, clear=True): + old_cwd = os.getcwd() + try: + os.chdir(worktree_path) + + explicit = Repo(os.getcwd()) + autodiscovered = Repo() + + self.assertTrue(osp.samefile(explicit.working_tree_dir, worktree_path)) + self.assertTrue(osp.samefile(autodiscovered.working_tree_dir, worktree_path)) + self.assertTrue(osp.samefile(autodiscovered.working_tree_dir, explicit.working_tree_dir)) + finally: + os.chdir(old_cwd) + @with_rw_directory def test_rebasing(self, rw_dir): r = Repo.init(rw_dir) From 25ba54dd3fb374b8fade7de4be1ac2ac84722190 Mon Sep 17 00:00:00 2001 From: "GPT 5.5" Date: Tue, 28 Apr 2026 09:17:31 +0800 Subject: [PATCH 1304/1392] prevent out-of-repo access when manipulating references. This previously made it possible to create, modify and delete files outside outside of the repository, which is a problem if inputs aren't trusted. Co-authored-by: Sebastian Thiel --- git/refs/log.py | 2 +- git/refs/remote.py | 5 ++- git/refs/symbolic.py | 37 +++++++++++++++--- test/test_refs.py | 91 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 9 deletions(-) diff --git a/git/refs/log.py b/git/refs/log.py index 4751cff99..037e143d5 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -213,7 +213,7 @@ def path(cls, ref: "SymbolicReference") -> str: :param ref: :class:`~git.refs.symbolic.SymbolicReference` instance """ - return osp.join(ref.repo.git_dir, "logs", to_native_path(ref.path)) + return to_native_path(ref._get_validated_reflog_path(ref.repo, ref.path)) @classmethod def iter_entries(cls, stream: Union[str, "BytesIO", mmap]) -> Iterator[RefLogEntry]: diff --git a/git/refs/remote.py b/git/refs/remote.py index b4f4f7b36..8244470b0 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -63,12 +63,13 @@ def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: # generally ignored in the refs/ folder. We don't though and delete remainders # manually. for ref in refs: + cls._check_ref_name_valid(ref.path) try: - os.remove(os.path.join(repo.common_dir, ref.path)) + os.remove(cls._get_validated_path(repo.common_dir, ref.path)) except OSError: pass try: - os.remove(os.path.join(repo.git_dir, ref.path)) + os.remove(cls._get_validated_path(repo.git_dir, ref.path)) except OSError: pass # END for each ref diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index 99af4f57c..020de5e13 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -110,6 +110,32 @@ def name(self) -> str: def abspath(self) -> PathLike: return join_path_native(_git_dir(self.repo, self.path), self.path) + @staticmethod + def _get_validated_path(base: PathLike, path: PathLike) -> str: + path = os.fspath(path) + base_path = os.path.realpath(os.fspath(base)) + abs_path = os.path.realpath(os.path.join(base_path, path)) + try: + common_path = os.path.commonpath([base_path, abs_path]) + except ValueError as e: + raise ValueError("Reference path %r escapes the repository" % path) from e + if os.path.normcase(common_path) != os.path.normcase(base_path): + raise ValueError("Reference path %r escapes the repository" % path) + return abs_path + + @classmethod + def _get_validated_ref_path(cls, repo: "Repo", path: PathLike) -> str: + """Return the absolute filesystem path for a ref after validating it.""" + cls._check_ref_name_valid(path) + ref_path = os.fspath(path) + return cls._get_validated_path(_git_dir(repo, ref_path), ref_path) + + @classmethod + def _get_validated_reflog_path(cls, repo: "Repo", path: PathLike) -> str: + """Return the absolute filesystem path for a reflog after validating it.""" + cls._check_ref_name_valid(path) + return cls._get_validated_path(os.path.join(repo.git_dir, "logs"), path) + @classmethod def _get_packed_refs_path(cls, repo: "Repo") -> str: return os.path.join(repo.common_dir, "packed-refs") @@ -485,7 +511,7 @@ def set_reference( # END handle non-existing # END retrieve old hexsha - fpath = self.abspath + fpath = self._get_validated_ref_path(self.repo, self.path) assure_directory_exists(fpath, is_file=True) lfd = LockedFD(fpath) @@ -632,7 +658,7 @@ def delete(cls, repo: "Repo", path: PathLike) -> None: Alternatively the symbolic reference to be deleted. """ full_ref_path = cls.to_full_path(path) - abs_path = os.path.join(repo.common_dir, full_ref_path) + abs_path = cls._get_validated_ref_path(repo, full_ref_path) if os.path.exists(abs_path): os.remove(abs_path) else: @@ -695,9 +721,8 @@ def _create( symbolic reference. Otherwise it will be resolved to the corresponding object and a detached symbolic reference will be created instead. """ - git_dir = _git_dir(repo, path) full_ref_path = cls.to_full_path(path) - abs_ref_path = os.path.join(git_dir, full_ref_path) + abs_ref_path = cls._get_validated_ref_path(repo, full_ref_path) # Figure out target data. target = reference @@ -789,8 +814,8 @@ def rename(self, new_path: PathLike, force: bool = False) -> "SymbolicReference" if self.path == new_path: return self - new_abs_path = os.path.join(_git_dir(self.repo, new_path), new_path) - cur_abs_path = os.path.join(_git_dir(self.repo, self.path), self.path) + new_abs_path = self._get_validated_ref_path(self.repo, new_path) + cur_abs_path = self._get_validated_ref_path(self.repo, self.path) if os.path.isfile(new_abs_path): if not force: # If they point to the same file, it's not an error. diff --git a/test/test_refs.py b/test/test_refs.py index 329515807..4337f35e1 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -18,6 +18,7 @@ RefLog, Reference, RemoteReference, + Repo, SymbolicReference, TagReference, ) @@ -29,6 +30,14 @@ class TestRefs(TestBase): + def _repo_with_initial_commit(self, base_dir): + repo_dir = base_dir / "repo" + repo = Repo.init(repo_dir) + (repo_dir / "file.txt").write_text("initial\n", encoding="utf-8") + repo.index.add(["file.txt"]) + repo.index.commit("initial") + return repo + def test_from_path(self): # Should be able to create any reference directly. for ref_type in (Reference, Head, TagReference, RemoteReference): @@ -648,6 +657,88 @@ def test_refs_outside_repo(self): ref_file_name = Path(ref_file.name).name self.assertRaises(BadName, self.rorepo.commit, f"../../{ref_file_name}") + def test_reference_create_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + repo = self._repo_with_initial_commit(base_dir) + outside_path = base_dir / "outside_write.txt" + + self.assertRaises(ValueError, Reference.create, repo, "../../../outside_write.txt", "HEAD") + assert not outside_path.exists() + + def test_symbolic_reference_create_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + repo = self._repo_with_initial_commit(base_dir) + outside_path = base_dir / "outside_write.txt" + + self.assertRaises(ValueError, SymbolicReference.create, repo, "../../outside_write.txt", "HEAD") + assert not outside_path.exists() + + def test_symbolic_reference_set_reference_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + repo = self._repo_with_initial_commit(base_dir) + outside_path = base_dir / "outside_write.txt" + + self.assertRaises(ValueError, SymbolicReference(repo, "../../outside_write.txt").set_reference, "HEAD") + assert not outside_path.exists() + + def test_symbolic_reference_rename_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + repo = self._repo_with_initial_commit(base_dir) + outside_path = base_dir / "outside_move.txt" + ref = SymbolicReference.create(repo, "SAFE_RENAME_SOURCE", "HEAD") + + self.assertRaises(ValueError, ref.rename, "../../outside_move.txt") + assert not outside_path.exists() + assert Path(ref.abspath).is_file() + + def test_symbolic_reference_delete_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + repo = self._repo_with_initial_commit(base_dir) + outside_path = base_dir / "outside_delete.txt" + outside_path.write_text("do not delete\n", encoding="utf-8") + + self.assertRaises(ValueError, SymbolicReference.delete, repo, "../../outside_delete.txt") + assert outside_path.read_text(encoding="utf-8") == "do not delete\n" + + def test_symbolic_reference_log_append_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + repo = self._repo_with_initial_commit(base_dir) + outside_path = base_dir / "outside_reflog.txt" + + ref = SymbolicReference(repo, "../../../outside_reflog.txt") + self.assertRaises(ValueError, ref.log_append, Commit.NULL_BIN_SHA, "do not write", repo.head.commit.binsha) + assert not outside_path.exists() + + def test_remote_reference_delete_cleanup_rejects_path_traversal(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + git_dir = base_dir / "repo" / ".git" + git_dir.mkdir(parents=True) + outside_path = base_dir / "outside_remote_delete.txt" + outside_path.write_text("do not delete\n", encoding="utf-8") + + class GitStub: + def branch(self, *args): + pass + + class RepoStub: + pass + + repo = RepoStub() + repo.git = GitStub() + repo.common_dir = str(git_dir) + repo.git_dir = str(git_dir) + ref = RemoteReference(repo, "../../outside_remote_delete.txt", check_path=False) + + self.assertRaises(ValueError, RemoteReference.delete, repo, ref) + assert outside_path.read_text(encoding="utf-8") == "do not delete\n" + def test_validity_ref_names(self): """Ensure ref names are checked for validity. From 4af8463cca31c2369312fcaa5309dfc30756c7b6 Mon Sep 17 00:00:00 2001 From: "GPT 5.5" Date: Tue, 28 Apr 2026 09:30:41 +0800 Subject: [PATCH 1305/1392] address review feedback and CI failures Consolidate follow-up fixes from review and CI: - fix lint and mypy issues in reference log path handling - validate remote reference paths before invoking git branch deletion - add symlink escape coverage where realpath resolves symlinks - ensure temporary test repositories release git resources during cleanup Co-authored-by: Sebastian Thiel --- git/refs/log.py | 4 +- git/refs/remote.py | 4 +- git/util.py | 2 +- test/test_refs.py | 92 +++++++++++++++++++++++++++++++--------------- 4 files changed, 69 insertions(+), 33 deletions(-) diff --git a/git/refs/log.py b/git/refs/log.py index 037e143d5..fbbe66b22 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -4,7 +4,6 @@ __all__ = ["RefLog", "RefLogEntry"] from mmap import mmap -import os.path as osp import re import time as _time @@ -212,6 +211,9 @@ def path(cls, ref: "SymbolicReference") -> str: :param ref: :class:`~git.refs.symbolic.SymbolicReference` instance + + :raise ValueError: + If `ref.path` is invalid or escapes the repository's reflog directory. """ return to_native_path(ref._get_validated_reflog_path(ref.repo, ref.path)) diff --git a/git/refs/remote.py b/git/refs/remote.py index 8244470b0..e16ae70f8 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -58,12 +58,14 @@ def delete(cls, repo: "Repo", *refs: "RemoteReference", **kwargs: Any) -> None: `kwargs` are given for comparability with the base class method as we should not narrow the signature. """ + for ref in refs: + cls._check_ref_name_valid(ref.path) + repo.git.branch("-d", "-r", *refs) # The official deletion method will ignore remote symbolic refs - these are # generally ignored in the refs/ folder. We don't though and delete remainders # manually. for ref in refs: - cls._check_ref_name_valid(ref.path) try: os.remove(cls._get_validated_path(repo.common_dir, ref.path)) except OSError: diff --git a/git/util.py b/git/util.py index c3ffdd62b..712fabe85 100644 --- a/git/util.py +++ b/git/util.py @@ -289,7 +289,7 @@ def join_path(a: PathLike, *p: PathLike) -> PathLike: if sys.platform == "win32": - def to_native_path_windows(path: PathLike) -> PathLike: + def to_native_path_windows(path: PathLike) -> str: path = os.fspath(path) return path.replace("/", "\\") diff --git a/test/test_refs.py b/test/test_refs.py index 4337f35e1..d77b34eba 100644 --- a/test/test_refs.py +++ b/test/test_refs.py @@ -3,6 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +import contextlib from itertools import chain import os.path as osp from pathlib import Path @@ -30,13 +31,17 @@ class TestRefs(TestBase): + @contextlib.contextmanager def _repo_with_initial_commit(self, base_dir): repo_dir = base_dir / "repo" repo = Repo.init(repo_dir) (repo_dir / "file.txt").write_text("initial\n", encoding="utf-8") repo.index.add(["file.txt"]) repo.index.commit("initial") - return repo + try: + yield repo + finally: + repo.git.clear_cache() def test_from_path(self): # Should be able to create any reference directly. @@ -660,60 +665,84 @@ def test_refs_outside_repo(self): def test_reference_create_rejects_path_traversal(self): with tempfile.TemporaryDirectory() as tmp_dir: base_dir = Path(tmp_dir) - repo = self._repo_with_initial_commit(base_dir) - outside_path = base_dir / "outside_write.txt" + with self._repo_with_initial_commit(base_dir) as repo: + outside_path = base_dir / "outside_write.txt" - self.assertRaises(ValueError, Reference.create, repo, "../../../outside_write.txt", "HEAD") - assert not outside_path.exists() + self.assertRaises(ValueError, Reference.create, repo, "../../../outside_write.txt", "HEAD") + assert not outside_path.exists() def test_symbolic_reference_create_rejects_path_traversal(self): with tempfile.TemporaryDirectory() as tmp_dir: base_dir = Path(tmp_dir) - repo = self._repo_with_initial_commit(base_dir) - outside_path = base_dir / "outside_write.txt" + with self._repo_with_initial_commit(base_dir) as repo: + outside_path = base_dir / "outside_write.txt" - self.assertRaises(ValueError, SymbolicReference.create, repo, "../../outside_write.txt", "HEAD") - assert not outside_path.exists() + self.assertRaises(ValueError, SymbolicReference.create, repo, "../../outside_write.txt", "HEAD") + assert not outside_path.exists() def test_symbolic_reference_set_reference_rejects_path_traversal(self): with tempfile.TemporaryDirectory() as tmp_dir: base_dir = Path(tmp_dir) - repo = self._repo_with_initial_commit(base_dir) - outside_path = base_dir / "outside_write.txt" + with self._repo_with_initial_commit(base_dir) as repo: + outside_path = base_dir / "outside_write.txt" - self.assertRaises(ValueError, SymbolicReference(repo, "../../outside_write.txt").set_reference, "HEAD") - assert not outside_path.exists() + self.assertRaises(ValueError, SymbolicReference(repo, "../../outside_write.txt").set_reference, "HEAD") + assert not outside_path.exists() def test_symbolic_reference_rename_rejects_path_traversal(self): with tempfile.TemporaryDirectory() as tmp_dir: base_dir = Path(tmp_dir) - repo = self._repo_with_initial_commit(base_dir) - outside_path = base_dir / "outside_move.txt" - ref = SymbolicReference.create(repo, "SAFE_RENAME_SOURCE", "HEAD") + with self._repo_with_initial_commit(base_dir) as repo: + outside_path = base_dir / "outside_move.txt" + ref = SymbolicReference.create(repo, "SAFE_RENAME_SOURCE", "HEAD") - self.assertRaises(ValueError, ref.rename, "../../outside_move.txt") - assert not outside_path.exists() - assert Path(ref.abspath).is_file() + self.assertRaises(ValueError, ref.rename, "../../outside_move.txt") + assert not outside_path.exists() + assert Path(ref.abspath).is_file() def test_symbolic_reference_delete_rejects_path_traversal(self): with tempfile.TemporaryDirectory() as tmp_dir: base_dir = Path(tmp_dir) - repo = self._repo_with_initial_commit(base_dir) - outside_path = base_dir / "outside_delete.txt" - outside_path.write_text("do not delete\n", encoding="utf-8") + with self._repo_with_initial_commit(base_dir) as repo: + outside_path = base_dir / "outside_delete.txt" + outside_path.write_text("do not delete\n", encoding="utf-8") - self.assertRaises(ValueError, SymbolicReference.delete, repo, "../../outside_delete.txt") - assert outside_path.read_text(encoding="utf-8") == "do not delete\n" + self.assertRaises(ValueError, SymbolicReference.delete, repo, "../../outside_delete.txt") + assert outside_path.read_text(encoding="utf-8") == "do not delete\n" def test_symbolic_reference_log_append_rejects_path_traversal(self): with tempfile.TemporaryDirectory() as tmp_dir: base_dir = Path(tmp_dir) - repo = self._repo_with_initial_commit(base_dir) - outside_path = base_dir / "outside_reflog.txt" + with self._repo_with_initial_commit(base_dir) as repo: + outside_path = base_dir / "outside_reflog.txt" + + ref = SymbolicReference(repo, "../../../outside_reflog.txt") + self.assertRaises( + ValueError, ref.log_append, Commit.NULL_BIN_SHA, "do not write", repo.head.commit.binsha + ) + assert not outside_path.exists() - ref = SymbolicReference(repo, "../../../outside_reflog.txt") - self.assertRaises(ValueError, ref.log_append, Commit.NULL_BIN_SHA, "do not write", repo.head.commit.binsha) - assert not outside_path.exists() + def test_symbolic_reference_set_reference_rejects_symlink_escape(self): + with tempfile.TemporaryDirectory() as tmp_dir: + base_dir = Path(tmp_dir) + with self._repo_with_initial_commit(base_dir) as repo: + outside_dir = base_dir / "outside_refs" + outside_dir.mkdir() + outside_path = outside_dir / "escaped" + + refs_heads_dir = Path(repo.common_dir) / "refs" / "heads" + refs_heads_dir.mkdir(parents=True, exist_ok=True) + symlink_path = refs_heads_dir / "link_out" + try: + symlink_path.symlink_to(outside_dir, target_is_directory=True) + except (OSError, NotImplementedError) as ex: + self.skipTest("symlinks unavailable on this platform: %s" % ex) + if osp.realpath(symlink_path / "escaped") == osp.abspath(symlink_path / "escaped"): + self.skipTest("realpath does not resolve directory symlinks on this platform") + + ref = SymbolicReference(repo, "refs/heads/link_out/escaped") + self.assertRaises(ValueError, ref.set_reference, "HEAD") + assert not outside_path.exists() def test_remote_reference_delete_cleanup_rejects_path_traversal(self): with tempfile.TemporaryDirectory() as tmp_dir: @@ -724,8 +753,10 @@ def test_remote_reference_delete_cleanup_rejects_path_traversal(self): outside_path.write_text("do not delete\n", encoding="utf-8") class GitStub: + branch_called = False + def branch(self, *args): - pass + self.branch_called = True class RepoStub: pass @@ -737,6 +768,7 @@ class RepoStub: ref = RemoteReference(repo, "../../outside_remote_delete.txt", check_path=False) self.assertRaises(ValueError, RemoteReference.delete, repo, ref) + assert not repo.git.branch_called assert outside_path.read_text(encoding="utf-8") == "do not delete\n" def test_validity_ref_names(self): From 5a15361e0e1223f5c2e2c05688e6d094796b954d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 28 Apr 2026 13:24:46 +0800 Subject: [PATCH 1306/1392] a new release with safer reference creation --- VERSION | 2 +- doc/source/changes.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e1ace7c6e..94c78f538 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.47 +3.1.48 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 90b2e0739..4ac67d077 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,15 @@ Changelog ========= +3.1.48 +====== + +Safe reference creation in the face of untrusted input. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.48 + + 3.1.47 ====== From c417af469f9aa3da8dfef78f996c0fb8c5d1f4c2 Mon Sep 17 00:00:00 2001 From: "GPT 5.5" Date: Wed, 29 Apr 2026 05:47:57 +0800 Subject: [PATCH 1307/1392] reject control chars in written values in configuration Reject CR, LF, and NUL in GitConfigParser values before writing them to git config files (which also is a deviation from Git which escapes them). GitConfigParser._write() serializes embedded newlines as indented continuation lines by replacing "\n" with "\n\t". Git itself skips leading whitespace before parsing config tokens, so an injected value such as: foo [core] hooksPath=/tmp/hooks is written in a form where the indented "[core]" line is still parsed by Git as a real section header. This lets attacker-controlled input passed to config_writer().set_value() poison repository config, including core.hooksPath, and redirect hook execution for later Git operations. Fail closed instead of stripping or normalizing these characters. Silent normalization can hide unsanitized caller input, and GitPython does not currently round-trip Git-style escaped values such as "\n" as embedded newlines. Apply the validation to set_value(), add_value(), and the public set() path so callers cannot bypass the safer helper API. Add regression tests for the advisory payload and for CR, LF, NUL, and bytes values. This preserves existing read behavior for config files that already contain multiline values while preventing GitPython from writing new unsafe values. Co-authored-by: Sebastian Thiel --- git/config.py | 24 ++++++++++++++++++++++-- test/test_config.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/git/config.py b/git/config.py index c6eaf8f7b..31d9e01cd 100644 --- a/git/config.py +++ b/git/config.py @@ -882,6 +882,24 @@ def _value_to_string(self, value: Union[str, bytes, int, float, bool]) -> str: return str(value) return force_text(value) + def _value_to_string_safe(self, value: Union[str, bytes, int, float, bool]) -> str: + value_str = self._value_to_string(value) + if re.search(r"[\r\n\x00]", value_str): + raise ValueError("Git config values must not contain CR, LF, or NUL") + return value_str + + @needs_values + @set_dirty_and_flush_changes + def set( + self, + section: str, + option: str, + value: Union[str, bytes, int, float, bool, None] = None, + ) -> None: + if value is not None: + value = self._value_to_string_safe(value) + return super().set(section, option, value) + @needs_values @set_dirty_and_flush_changes def set_value(self, section: str, option: str, value: Union[str, bytes, int, float, bool]) -> "GitConfigParser": @@ -902,9 +920,10 @@ def set_value(self, section: str, option: str, value: Union[str, bytes, int, flo :return: This instance """ + value_str = self._value_to_string_safe(value) if not self.has_section(section): self.add_section(section) - self.set(section, option, self._value_to_string(value)) + self.set(section, option, value_str) return self @needs_values @@ -929,9 +948,10 @@ def add_value(self, section: str, option: str, value: Union[str, bytes, int, flo :return: This instance """ + value_str = self._value_to_string_safe(value) if not self.has_section(section): self.add_section(section) - self._sections[section].add(option, self._value_to_string(value)) + self._sections[section].add(option, value_str) return self def rename_section(self, section: str, new_name: str) -> "GitConfigParser": diff --git a/test/test_config.py b/test/test_config.py index 11ea52d16..a9dcdb087 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -150,6 +150,39 @@ def test_config_value_with_trailing_new_line(self): git_config = GitConfigParser(config_file) git_config.read() # This should not throw an exception + @with_rw_directory + def test_set_value_rejects_config_injection(self, rw_dir): + config_path = osp.join(rw_dir, "config") + payload = "foo\n[core]\nhooksPath=/tmp/hooks" + + with GitConfigParser(config_path, read_only=False) as git_config: + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set_value("user", "name", payload) + + with GitConfigParser(config_path, read_only=True) as git_config: + self.assertFalse(git_config.has_section("user")) + self.assertFalse(git_config.has_section("core")) + + @with_rw_directory + def test_set_and_add_value_reject_unsafe_value_characters(self, rw_dir): + config_path = osp.join(rw_dir, "config") + bad_values = ("foo\rbar", "foo\nbar", "foo\x00bar", b"foo\nbar") + + with GitConfigParser(config_path, read_only=False) as git_config: + git_config.add_section("user") + for bad_value in bad_values: + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set("user", "name", bad_value) + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set_value("user", "name", bad_value) + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.add_value("user", "name", bad_value) + + git_config.set_value("user", "name", "safe") + + with GitConfigParser(config_path, read_only=True) as git_config: + self.assertEqual(git_config.get_value("user", "name"), "safe") + def test_base(self): path_repo = fixture_path("git_config") path_global = fixture_path("git_config_global") From 8e24503b42c1d63dd98e8b2e6a2f655bdd0821e3 Mon Sep 17 00:00:00 2001 From: "GPT 5.5" Date: Wed, 29 Apr 2026 06:39:02 +0800 Subject: [PATCH 1308/1392] avoid duplicate validation in set_value Co-authored-by: Sebastian Thiel --- git/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/config.py b/git/config.py index 31d9e01cd..97ae054e5 100644 --- a/git/config.py +++ b/git/config.py @@ -923,7 +923,7 @@ def set_value(self, section: str, option: str, value: Union[str, bytes, int, flo value_str = self._value_to_string_safe(value) if not self.has_section(section): self.add_section(section) - self.set(section, option, value_str) + super().set(section, option, value_str) return self @needs_values From d7ce6fc19199cf8698d722c7d8ae38ff81424fba Mon Sep 17 00:00:00 2001 From: "GPT 5.5" Date: Tue, 28 Apr 2026 21:47:27 +0000 Subject: [PATCH 1309/1392] Improve pure Python rev-parse coverage and behavior (#2135) Port object-resolving revspec cases inspired by gix-revision into deterministic GitPython tests, without shelling out to Git or Gix at runtime. Refactor rev_parse handling around anchors, navigation, peeling, reflog selectors, path/index lookups, describe-style names, and commit-message searches. Document observed Git/Gix behavior differences and the GitPython choices made for user-facing compatibility. Co-authored-by: Sebastian Thiel --- git/repo/fun.py | 513 ++++++++++++++++++++++++++++++----------- test/test_repo.py | 17 ++ test/test_rev_parse.py | 138 +++++++++++ 3 files changed, 536 insertions(+), 132 deletions(-) create mode 100644 test/test_rev_parse.py diff --git a/git/repo/fun.py b/git/repo/fun.py index 3f00e60ea..d91ce5c0b 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -20,6 +20,7 @@ import os import os.path as osp from pathlib import Path +import re import stat from string import digits @@ -28,12 +29,13 @@ from git.cmd import Git from git.exc import WorkTreeRepositoryUnsupported from git.objects import Object +from git.objects.util import parse_date from git.refs import SymbolicReference from git.util import cygpath, bin_to_hex, hex_to_bin # Typing ---------------------------------------------------------------------- -from typing import Optional, TYPE_CHECKING, Union, cast, overload +from typing import Optional, TYPE_CHECKING, Tuple, Union, cast, overload from git.types import AnyGitObject, Literal, PathLike @@ -41,6 +43,7 @@ from git.db import GitCmdObjectDB from git.objects import Commit, TagObject from git.refs.reference import Reference + from git.refs.log import RefLog, RefLogEntry from git.refs.tag import Tag from .base import Repo @@ -139,6 +142,23 @@ def short_to_long(odb: "GitCmdObjectDB", hexsha: str) -> Optional[bytes]: # END exception handling +def _describe_to_long(repo: "Repo", name: str) -> Optional[bytes]: + """Resolve git-describe style names to the abbreviated object they contain.""" + match = re.match(r"^.+-\d+-g([0-9A-Fa-f]{4,40})(?:-dirty)?$", name) + if match is None: + match = re.match(r"^.+-g([0-9A-Fa-f]{4,40})(?:-dirty)?$", name) + if match is None: + match = re.match(r"^([0-9A-Fa-f]{4,40})-dirty$", name) + if match is None: + return None + # END handle match + + hexsha = match.group(1) + if len(hexsha) == 40: + return hexsha.encode("ascii") + return short_to_long(repo.odb, hexsha) + + @overload def name_to_object(repo: "Repo", name: str, return_ref: Literal[False] = ...) -> AnyGitObject: ... @@ -170,6 +190,10 @@ def name_to_object(repo: "Repo", name: str, return_ref: bool = False) -> Union[A # END handle short shas # END find sha if it matches + if hexsha is None: + hexsha = _describe_to_long(repo, name) + # END handle describe output + # If we couldn't find an object for what seemed to be a short hexsha, try to find it # as reference anyway, it could be named 'aaa' for instance. if hexsha is None: @@ -227,6 +251,298 @@ def to_commit(obj: Object) -> "Commit": return obj +def _object_from_hexsha(repo: "Repo", hexsha: str) -> AnyGitObject: + return Object.new_from_sha(repo, hex_to_bin(hexsha)) + + +def _current_reflog_ref(repo: "Repo") -> SymbolicReference: + return repo.head + + +def _ref_log(repo: "Repo", ref: SymbolicReference) -> "RefLog": + try: + return ref.log() + except FileNotFoundError: + try: + if ref.path == repo.head.ref.path: + return repo.head.log() + # END handle linked-worktree current branch logs + except TypeError: + pass + # END handle detached head + raise + # END handle missing branch log + + +def _ref_log_entry(repo: "Repo", ref: SymbolicReference, index: int) -> "RefLogEntry": + try: + return ref.log_entry(index) + except FileNotFoundError: + try: + if ref.path == repo.head.ref.path: + return repo.head.log_entry(index) + # END handle linked-worktree current branch logs + except TypeError: + pass + # END handle detached head + raise + # END handle missing branch log + + +def _find_reflog_entry_by_date(repo: "Repo", ref: SymbolicReference, spec: str) -> str: + try: + timestamp, _offset = parse_date(spec) + except ValueError as e: + raise NotImplementedError("Support for additional @{...} modes not implemented") from e + # END handle unsupported dates + log = _ref_log(repo, ref) + if not log: + raise IndexError("Invalid revlog date: %s" % spec) + # END handle empty log + + for entry in reversed(log): + if entry.time[0] <= timestamp: + return entry.newhexsha + # END found candidate + # END for each entry + return log[0].newhexsha + + +def _previous_checked_out_branch(repo: "Repo", nth: int) -> AnyGitObject: + if nth <= 0: + raise ValueError("Invalid previous checkout selector: -%i" % nth) + # END handle invalid input + + seen = 0 + for entry in reversed(_ref_log(repo, repo.head)): + message = entry.message or "" + prefix = "checkout: moving from " + if not message.startswith(prefix): + continue + # END skip non-checkouts + + previous_branch = message[len(prefix) :].split(" to ", 1)[0] + seen += 1 + if seen == nth: + return name_to_object(repo, previous_branch) + # END found selector + # END for each entry + raise IndexError("Invalid previous checkout selector: -%i" % nth) + + +def _tracking_branch_object(repo: "Repo", ref: Optional[SymbolicReference]) -> AnyGitObject: + from git.refs.head import Head + + if ref is None: + try: + head = repo.active_branch + except TypeError as e: + raise BadName("@{upstream}") from e + elif isinstance(ref, Head): + head = ref + else: + raise BadName("%s@{upstream}" % ref.name) + # END handle head + + tracking_branch = head.tracking_branch() + if tracking_branch is None: + raise BadName("%s@{upstream}" % head.name) + # END handle missing upstream + return tracking_branch.commit + + +def _apply_reflog(repo: "Repo", ref: Optional[SymbolicReference], content: str) -> AnyGitObject: + if content.startswith("+"): + content = content[1:] + # END handle explicit positive sign + + if content.startswith("-"): + if ref is not None: + raise ValueError("Previous checkout selectors do not take an explicit ref") + if content == "-0": + raise ValueError("Negative zero is invalid in reflog selector") + # END handle invalid negative zero + try: + return _previous_checked_out_branch(repo, int(content[1:])) + except ValueError as e: + raise ValueError("Invalid previous checkout selector: %s" % content) from e + # END handle previous checkout branch + + content_lower = content.lower() + if content_lower in ("u", "upstream", "push"): + return _tracking_branch_object(repo, ref) + # END handle sibling branches + + ref = ref or _current_reflog_ref(repo) + try: + entry_no = int(content) + except ValueError: + hexsha = _find_reflog_entry_by_date(repo, ref, content) + else: + if entry_no >= 100000000: + hexsha = _find_reflog_entry_by_date(repo, ref, "%s +0000" % entry_no) + elif entry_no == 0: + return ref.commit + else: + try: + entry = _ref_log_entry(repo, ref, -(entry_no + 1)) + except IndexError as e: + raise IndexError("Invalid revlog index: %i" % entry_no) from e + # END handle index out of bound + hexsha = entry.newhexsha + # END handle offset or date-like timestamp + # END handle content + return _object_from_hexsha(repo, hexsha) + + +def _find_closing_brace(rev: str, start: int) -> int: + depth = 1 + escaped = False + for idx in range(start + 1, len(rev)): + char = rev[idx] + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + return idx + # END found end + # END handle char + # END for each char + raise ValueError("Missing closing brace to define type in %s" % rev) + + +def _parse_search(pattern: str) -> Tuple[str, bool]: + if not pattern: + raise ValueError("Revision search requires a pattern") + # END handle empty pattern + + if pattern.startswith("!-"): + return pattern[2:], True + if pattern.startswith("!!"): + return pattern[1:], False + if pattern.startswith("!"): + raise ValueError("Need one character after /!, typically -") + return pattern, False + + +def _unescape_braced_regex(pattern: str) -> str: + out = [] + idx = 0 + while idx < len(pattern): + char = pattern[idx] + if char == "\\" and idx + 1 < len(pattern): + next_char = pattern[idx + 1] + if next_char in "{}\\": + out.append(next_char) + else: + out.append(char) + out.append(next_char) + # END handle escaped char + idx += 2 + continue + # END handle backslash + out.append(char) + idx += 1 + # END for each char + return "".join(out) + + +def _find_commit_by_message( + repo: "Repo", rev: Optional[AnyGitObject], pattern: str, braced: bool = False +) -> AnyGitObject: + pattern, negated = _parse_search(_unescape_braced_regex(pattern) if braced else pattern) + regex = re.compile(pattern) + if rev is None: + commits = repo.iter_commits("--all") + else: + commits = repo.iter_commits(to_commit(cast(Object, rev)).hexsha) + # END handle starting point + + for commit in commits: + matches = regex.search(commit.message or "") is not None + if matches != negated: + return commit + # END found commit + # END for each commit + raise BadName("No commit found matching message pattern %r" % pattern) + + +def _index_lookup(repo: "Repo", spec: str) -> AnyGitObject: + if not spec: + raise ValueError("':' must be followed by a path") + # END handle empty lookup + + stage = 0 + path = spec + if len(spec) >= 2 and spec[1] == ":" and spec[0] in "0123": + stage = int(spec[0]) + path = spec[2:] + # END handle stage + + try: + return repo.index.entries[(path, stage)].to_blob(repo) + except KeyError as e: + raise BadName("Path %r did not exist in the index at stage %i" % (path, stage)) from e + + +def _tree_lookup(obj: AnyGitObject, path: str) -> AnyGitObject: + if obj.type != "tree": + obj = to_commit(cast(Object, obj)).tree + # END get tree + if not path: + return obj + return obj[path] + + +def _peel(obj: AnyGitObject, output_type: str, repo: "Repo", rev: str) -> AnyGitObject: + if output_type == "/": + return obj + if output_type.startswith("/"): + return _find_commit_by_message(repo, obj, output_type[1:], braced=True) + if output_type == "": + return deref_tag(cast("TagObject", obj)) if obj.type == "tag" else obj + if output_type == "object": + return obj + if output_type == "commit": + return to_commit(cast(Object, obj)) + if output_type == "tree": + return to_commit(cast(Object, obj)).tree if obj.type != "tree" else obj + if output_type == "blob": + obj = deref_tag(cast("TagObject", obj)) if obj.type == "tag" else obj + if obj.type == output_type: + return obj + # END handle matching type + raise ValueError("Could not accommodate requested object type %r, got %s" % (output_type, obj.type)) + if output_type == "tag": + if obj.type == output_type: + return obj + # END handle matching type + raise ValueError("Could not accommodate requested object type %r, got %s" % (output_type, obj.type)) + # END handle known types + raise ValueError("Invalid output type: %s ( in %s )" % (output_type, rev)) + + +def _first_rev_token(rev: str) -> Optional[int]: + for idx, char in enumerate(rev): + if char in "^~:": + return idx + if char == "@": + next_char = rev[idx + 1] if idx + 1 < len(rev) else None + if idx == 0 and next_char in (None, "^", "~", ":", "{"): + return idx + if next_char == "{": + return idx + # END handle reflog selector + # END handle at symbol + # END for each char + return None + + def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: """Parse a revision string. Like :manpage:`git-rev-parse(1)`. @@ -253,135 +569,81 @@ def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: :raise IndexError: If an invalid reflog index is specified. """ - # Are we in colon search mode? if rev.startswith(":/"): - # Colon search mode - raise NotImplementedError("commit by message search (regex)") - # END handle search + return _find_commit_by_message(repo, None, rev[2:]) + if rev.startswith(":"): + return _index_lookup(repo, rev[1:]) + # END handle top-level colon modes obj: Optional[AnyGitObject] = None ref = None - output_type = "commit" - start = 0 - parsed_to = 0 lr = len(rev) - while start < lr: - if rev[start] not in "^~:@": - start += 1 - continue - # END handle start + first_token = _first_rev_token(rev) + if first_token is None: + return name_to_object(repo, rev) + # END handle plain name + + if first_token == 0: + if rev[0] != "@": + raise ValueError("Revision specifier must start with an object name: %s" % rev) + # END handle invalid leading token + ref = _current_reflog_ref(repo) + obj = ref.commit + start = 0 if rev.startswith("@{") else 1 + else: + if rev[first_token] == "@": + ref = cast("Reference", name_to_object(repo, rev[:first_token], return_ref=True)) + obj = ref.commit + else: + obj = name_to_object(repo, rev[:first_token]) + # END handle anchor + start = first_token + # END initialize anchor + while start < lr: token = rev[start] - if obj is None: - # token is a rev name. - if start == 0: - ref = repo.head.ref - else: - if token == "@": - ref = cast("Reference", name_to_object(repo, rev[:start], return_ref=True)) - else: - obj = name_to_object(repo, rev[:start]) - # END handle token - # END handle refname - else: - if ref is not None: - obj = ref.commit - # END handle ref - # END initialize obj on first token - - start += 1 + if token == "@": + if start + 1 >= lr or rev[start + 1] != "{": + raise ValueError("Invalid @ token in revision specifier: %s" % rev) + # END handle invalid @ + end = _find_closing_brace(rev, start + 1) + obj = _apply_reflog(repo, ref if first_token != 0 and start == first_token else None, rev[start + 2 : end]) + ref = None + start = end + 1 + continue + # END handle reflog - # Try to parse {type}. - if start < lr and rev[start] == "{": - end = rev.find("}", start) - if end == -1: - raise ValueError("Missing closing brace to define type in %s" % rev) - output_type = rev[start + 1 : end] # Exclude brace. - - # Handle type. - if output_type == "commit": - obj = cast("TagObject", obj) - if obj and obj.type == "tag": - obj = deref_tag(obj) - else: - # Cannot do anything for non-tags. - pass - # END handle tag - elif output_type == "tree": - try: - obj = cast(AnyGitObject, obj) - obj = to_commit(obj).tree - except (AttributeError, ValueError): - pass # Error raised later. - # END exception handling - elif output_type in ("", "blob"): - obj = cast("TagObject", obj) - if obj and obj.type == "tag": - obj = deref_tag(obj) - else: - # Cannot do anything for non-tags. - pass - # END handle tag - elif token == "@": - # try single int - assert ref is not None, "Require Reference to access reflog" - revlog_index = None - try: - # Transform reversed index into the format of our revlog. - revlog_index = -(int(output_type) + 1) - except ValueError as e: - # TODO: Try to parse the other date options, using parse_date maybe. - raise NotImplementedError("Support for additional @{...} modes not implemented") from e - # END handle revlog index - - try: - entry = ref.log_entry(revlog_index) - except IndexError as e: - raise IndexError("Invalid revlog index: %i" % revlog_index) from e - # END handle index out of bound - - obj = Object.new_from_sha(repo, hex_to_bin(entry.newhexsha)) - - # Make it pass the following checks. - output_type = "" - else: - raise ValueError("Invalid output type: %s ( in %s )" % (output_type, rev)) - # END handle output type + if token == ":": + return _tree_lookup(cast(AnyGitObject, obj), rev[start + 1 :]) + # END handle path - # Empty output types don't require any specific type, its just about - # dereferencing tags. - if output_type and obj and obj.type != output_type: - raise ValueError("Could not accommodate requested object type %r, got %s" % (output_type, obj.type)) - # END verify output type + start += 1 - start = end + 1 # Skip brace. - parsed_to = start + if token == "^" and start < lr and rev[start] == "{": + end = _find_closing_brace(rev, start) + obj = _peel(cast(AnyGitObject, obj), rev[start + 1 : end], repo, rev) + ref = None + start = end + 1 continue # END parse type - # Try to parse a number. num = 0 - if token != ":": - found_digit = False - while start < lr: - if rev[start] in digits: - num = num * 10 + int(rev[start]) - start += 1 - found_digit = True - else: - break - # END handle number - # END number parse loop - - # No explicit number given, 1 is the default. It could be 0 though. - if not found_digit: - num = 1 - # END set default num - # END number parsing only if non-blob mode - - parsed_to = start - # Handle hierarchy walk. + found_digit = False + while start < lr: + if rev[start] in digits: + num = num * 10 + int(rev[start]) + start += 1 + found_digit = True + else: + break + # END handle number + # END number parse loop + + if not found_digit: + num = 1 + # END set default num + try: obj = cast(AnyGitObject, obj) if token == "~": @@ -391,15 +653,11 @@ def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: # END for each history item to walk elif token == "^": obj = to_commit(obj) - # Must be n'th parent. - if num: + if num == 0: + pass + else: obj = obj.parents[num - 1] - elif token == ":": - if obj.type != "tree": - obj = obj.tree - # END get tree type - obj = obj[rev[start:]] - parsed_to = lr + # END handle parent else: raise ValueError("Invalid token: %r" % token) # END end handle tag @@ -410,16 +668,7 @@ def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: # END exception handling # END parse loop - # Still no obj? It's probably a simple name. - if obj is None: - obj = name_to_object(repo, rev) - parsed_to = lr - # END handle simple name - if obj is None: raise ValueError("Revision specifier could not be parsed: %s" % rev) - if parsed_to != lr: - raise ValueError("Didn't consume complete rev spec %s, consumed part: %s" % (rev, rev[:parsed_to])) - return obj diff --git a/test/test_repo.py b/test/test_repo.py index 544b5c561..0dd3d5945 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -146,6 +146,23 @@ def test_commit_from_revision(self): self.assertEqual(commit.type, "commit") self.assertEqual(self.rorepo.commit(commit), commit) + @with_rw_directory + def test_commit_from_tag_starting_with_at(self, rw_dir): + repo = Repo.init(rw_dir) + with repo.config_writer() as writer: + writer.set_value("user", "name", "GitPython Tests") + writer.set_value("user", "email", "gitpython@example.com") + + tracked_file = Path(rw_dir) / "hello.txt" + tracked_file.write_text("hello") + repo.index.add([str(tracked_file)]) + commit = repo.index.commit("init") + repo.create_tag("@foo") + + self.assertEqual(repo.tags["@foo"].commit, commit) + self.assertEqual(repo.commit("@"), commit) + self.assertEqual(repo.commit("@foo"), commit) + def test_commits(self): mc = 10 commits = list(self.rorepo.iter_commits("0.1.6", max_count=mc)) diff --git a/test/test_rev_parse.py b/test/test_rev_parse.py new file mode 100644 index 000000000..371210fa9 --- /dev/null +++ b/test/test_rev_parse.py @@ -0,0 +1,138 @@ +from pathlib import Path + +import pytest + +from git import Repo +from gitdb.exc import BadName + + +def _write(repo, path, content): + full_path = Path(repo.working_tree_dir) / path + full_path.parent.mkdir(parents=True, exist_ok=True) + full_path.write_text(content) + repo.index.add([str(full_path)]) + + +@pytest.fixture +def rev_parse_repo(tmp_path): + repo = Repo.init(tmp_path) + with repo.config_writer() as writer: + writer.set_value("user", "name", "GitPython Tests") + writer.set_value("user", "email", "gitpython@example.com") + + _write(repo, "README.md", "root\n") + _write(repo, "CHANGES", "root changes\n") + _write(repo, "dir/file.txt", "root file\n") + root = repo.index.commit("root commit") + repo.create_tag("ann", ref=root, message="annotated tag") + + _write(repo, "README.md", "release\n") + release = repo.index.commit("release candidate") + repo.create_tag("v1.0", ref=release) + main = repo.active_branch + + side = repo.create_head("side", root) + side.checkout() + _write(repo, "side.txt", "side\n") + side_commit = repo.index.commit("side branch") + + main.checkout() + repo.git.merge("--no-ff", "side", "-m", "merge side") + merge = repo.head.commit + + repo.create_head("aaaaaaaa", merge) + repo.create_tag("@foo", ref=merge) + + return { + "repo": repo, + "root": root, + "release": release, + "side": side_commit, + "merge": merge, + "main": main, + } + + +def test_rev_parse_names_hex_and_describe_forms(rev_parse_repo): + repo = rev_parse_repo["repo"] + merge = rev_parse_repo["merge"] + + assert repo.rev_parse("@") == merge + assert repo.rev_parse("@foo") == merge + assert repo.rev_parse("aaaaaaaa") == merge + assert repo.rev_parse(merge.hexsha[:7]) == merge + assert repo.rev_parse("v1.0-1-g%s" % merge.hexsha[:7]) == merge + assert repo.rev_parse("anything-9-g%s" % merge.hexsha[:7]) == merge + assert repo.rev_parse("%s-dirty" % merge.hexsha[:7]) == merge + + +def test_rev_parse_navigation_and_peeling(rev_parse_repo): + repo = rev_parse_repo["repo"] + root = rev_parse_repo["root"] + release = rev_parse_repo["release"] + side = rev_parse_repo["side"] + merge = rev_parse_repo["merge"] + tag = repo.rev_parse("ann") + + assert repo.rev_parse("HEAD^0") == merge + assert repo.rev_parse("HEAD~0") == merge + assert repo.rev_parse("HEAD^1") == release + assert repo.rev_parse("HEAD^2") == side + assert repo.rev_parse("HEAD~") == release + assert repo.rev_parse("HEAD^^") == root + + assert tag.type == "tag" + assert repo.rev_parse("ann^{object}") == tag + assert repo.rev_parse("ann^{tag}") == tag + assert repo.rev_parse("ann^{}") == root + assert repo.rev_parse("ann^{commit}") == root + assert repo.rev_parse("HEAD^{tree}") == merge.tree + assert repo.rev_parse("HEAD^{/}") == merge + + +def test_rev_parse_tree_and_index_paths(rev_parse_repo): + repo = rev_parse_repo["repo"] + merge = rev_parse_repo["merge"] + + assert repo.rev_parse("HEAD:") == merge.tree + assert repo.rev_parse("HEAD:README.md") == merge.tree["README.md"] + assert repo.rev_parse("HEAD^{tree}:README.md") == merge.tree["README.md"] + assert repo.rev_parse(":README.md").binsha == merge.tree["README.md"].binsha + assert repo.rev_parse(":0:README.md").binsha == merge.tree["README.md"].binsha + + +def test_rev_parse_reflog_selectors(rev_parse_repo): + repo = rev_parse_repo["repo"] + merge = rev_parse_repo["merge"] + side = rev_parse_repo["side"] + main = rev_parse_repo["main"] + + assert repo.rev_parse("@{0}") == merge + assert repo.rev_parse("@{+0}") == merge + assert repo.rev_parse("%s@{0}" % main.name) == merge + assert repo.rev_parse("@{-1}") == side + + +def test_rev_parse_commit_message_search(rev_parse_repo): + repo = rev_parse_repo["repo"] + release = rev_parse_repo["release"] + merge = rev_parse_repo["merge"] + + assert repo.rev_parse(":/release") == release + assert repo.rev_parse("HEAD^{/release}") == release + assert repo.rev_parse("HEAD^{/!-release}") == merge + + +def test_rev_parse_rejects_invalid_object_specs(rev_parse_repo): + repo = rev_parse_repo["repo"] + + with pytest.raises(ValueError): + repo.rev_parse(":") + with pytest.raises(ValueError): + repo.rev_parse(":/") + with pytest.raises(ValueError): + repo.rev_parse("@{-0}") + with pytest.raises(ValueError): + repo.rev_parse("HEAD^{invalid}") + with pytest.raises(BadName): + repo.rev_parse(":missing") From bdbdf4bba08f59042a2e1197313ca9a2060021d0 Mon Sep 17 00:00:00 2001 From: Codex GPT-5 Date: Wed, 29 Apr 2026 06:55:03 +0800 Subject: [PATCH 1310/1392] Fix rev-parse CI issues --- git/repo/fun.py | 42 ++++++++++++++++++++++++++++++++++-------- test/test_repo.py | 9 +++++++-- test/test_rev_parse.py | 2 ++ 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/git/repo/fun.py b/git/repo/fun.py index d91ce5c0b..ed00dd833 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -41,7 +41,7 @@ if TYPE_CHECKING: from git.db import GitCmdObjectDB - from git.objects import Commit, TagObject + from git.objects import Commit from git.refs.reference import Reference from git.refs.log import RefLog, RefLogEntry from git.refs.tag import Tag @@ -256,13 +256,30 @@ def _object_from_hexsha(repo: "Repo", hexsha: str) -> AnyGitObject: def _current_reflog_ref(repo: "Repo") -> SymbolicReference: - return repo.head + try: + return repo.head.ref + except TypeError: + return repo.head + # END handle detached head + + +def _common_reflog_path(repo: "Repo", ref: SymbolicReference) -> Optional[str]: + if repo.common_dir == repo.git_dir: + return None + # END handle normal repository + return SymbolicReference._get_validated_path(osp.join(repo.common_dir, "logs"), ref.path) def _ref_log(repo: "Repo", ref: SymbolicReference) -> "RefLog": try: return ref.log() except FileNotFoundError: + common_path = _common_reflog_path(repo, ref) + if common_path and osp.isfile(common_path): + from git.refs.log import RefLog + + return RefLog.from_file(common_path) + # END handle linked-worktree branch logs try: if ref.path == repo.head.ref.path: return repo.head.log() @@ -278,6 +295,12 @@ def _ref_log_entry(repo: "Repo", ref: SymbolicReference, index: int) -> "RefLogE try: return ref.log_entry(index) except FileNotFoundError: + common_path = _common_reflog_path(repo, ref) + if common_path and osp.isfile(common_path): + from git.refs.log import RefLog + + return RefLog.entry_at(common_path, index) + # END handle linked-worktree branch logs try: if ref.path == repo.head.ref.path: return repo.head.log_entry(index) @@ -464,7 +487,11 @@ def _find_commit_by_message( # END handle starting point for commit in commits: - matches = regex.search(commit.message or "") is not None + message = commit.message + if isinstance(message, bytes): + message = message.decode(commit.encoding, "replace") + # END handle bytes message + matches = regex.search(message or "") is not None if matches != negated: return commit # END found commit @@ -505,7 +532,7 @@ def _peel(obj: AnyGitObject, output_type: str, repo: "Repo", rev: str) -> AnyGit if output_type.startswith("/"): return _find_commit_by_message(repo, obj, output_type[1:], braced=True) if output_type == "": - return deref_tag(cast("TagObject", obj)) if obj.type == "tag" else obj + return deref_tag(obj) if obj.type == "tag" else obj if output_type == "object": return obj if output_type == "commit": @@ -513,7 +540,7 @@ def _peel(obj: AnyGitObject, output_type: str, repo: "Repo", rev: str) -> AnyGit if output_type == "tree": return to_commit(cast(Object, obj)).tree if obj.type != "tree" else obj if output_type == "blob": - obj = deref_tag(cast("TagObject", obj)) if obj.type == "tag" else obj + obj = deref_tag(obj) if obj.type == "tag" else obj if obj.type == output_type: return obj # END handle matching type @@ -615,14 +642,14 @@ def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: # END handle reflog if token == ":": - return _tree_lookup(cast(AnyGitObject, obj), rev[start + 1 :]) + return _tree_lookup(obj, rev[start + 1 :]) # END handle path start += 1 if token == "^" and start < lr and rev[start] == "{": end = _find_closing_brace(rev, start) - obj = _peel(cast(AnyGitObject, obj), rev[start + 1 : end], repo, rev) + obj = _peel(obj, rev[start + 1 : end], repo, rev) ref = None start = end + 1 continue @@ -645,7 +672,6 @@ def rev_parse(repo: "Repo", rev: str) -> AnyGitObject: # END set default num try: - obj = cast(AnyGitObject, obj) if token == "~": obj = to_commit(obj) for _ in range(num): diff --git a/test/test_repo.py b/test/test_repo.py index 0dd3d5945..7262395bd 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -865,8 +865,13 @@ def test_rev_parse(self): # Currently, nothing more is supported. self.assertRaises(NotImplementedError, rev_parse, "@{1 week ago}") - # The last position. - assert rev_parse("@{1}") != head.commit + # The previous position, if this checkout has enough reflog history. + try: + previous = rev_parse("@{1}") + except IndexError: + pass + else: + self.assertNotEqual(previous, head.commit) def test_repo_odbtype(self): target_type = GitCmdObjectDB diff --git a/test/test_rev_parse.py b/test/test_rev_parse.py index 371210fa9..d96fdc1a2 100644 --- a/test/test_rev_parse.py +++ b/test/test_rev_parse.py @@ -106,9 +106,11 @@ def test_rev_parse_reflog_selectors(rev_parse_repo): merge = rev_parse_repo["merge"] side = rev_parse_repo["side"] main = rev_parse_repo["main"] + release = rev_parse_repo["release"] assert repo.rev_parse("@{0}") == merge assert repo.rev_parse("@{+0}") == merge + assert repo.rev_parse("@{1}") == release assert repo.rev_parse("%s@{0}" % main.name) == merge assert repo.rev_parse("@{-1}") == side From 6cf7ac33d449db095e8c301abba664836c16bfc8 Mon Sep 17 00:00:00 2001 From: Codex GPT-5 Date: Wed, 29 Apr 2026 07:11:05 +0800 Subject: [PATCH 1311/1392] Address rev-parse review feedback --- git/repo/fun.py | 56 ++++++++++++++++++++++++++++++++++-------- test/test_rev_parse.py | 35 ++++++++++++++++++++------ 2 files changed, 73 insertions(+), 18 deletions(-) diff --git a/git/repo/fun.py b/git/repo/fun.py index ed00dd833..66e7eba69 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -35,7 +35,7 @@ # Typing ---------------------------------------------------------------------- -from typing import Optional, TYPE_CHECKING, Tuple, Union, cast, overload +from typing import Iterator, Optional, TYPE_CHECKING, Tuple, Union, cast, overload from git.types import AnyGitObject, Literal, PathLike @@ -190,10 +190,6 @@ def name_to_object(repo: "Repo", name: str, return_ref: bool = False) -> Union[A # END handle short shas # END find sha if it matches - if hexsha is None: - hexsha = _describe_to_long(repo, name) - # END handle describe output - # If we couldn't find an object for what seemed to be a short hexsha, try to find it # as reference anyway, it could be named 'aaa' for instance. if hexsha is None: @@ -216,6 +212,10 @@ def name_to_object(repo: "Repo", name: str, return_ref: bool = False) -> Union[A # END for each base # END handle hexsha + if hexsha is None: + hexsha = _describe_to_long(repo, name) + # END handle describe output + # Didn't find any ref, this is an error. if return_ref: raise BadObject("Couldn't find reference named %r" % name) @@ -363,6 +363,8 @@ def _tracking_branch_object(repo: "Repo", ref: Optional[SymbolicReference]) -> A raise BadName("@{upstream}") from e elif isinstance(ref, Head): head = ref + elif os.fspath(ref.path).startswith("refs/heads/"): + head = Head(repo, ref.path) else: raise BadName("%s@{upstream}" % ref.name) # END handle head @@ -479,11 +481,15 @@ def _find_commit_by_message( repo: "Repo", rev: Optional[AnyGitObject], pattern: str, braced: bool = False ) -> AnyGitObject: pattern, negated = _parse_search(_unescape_braced_regex(pattern) if braced else pattern) - regex = re.compile(pattern) + try: + regex = re.compile(pattern) + except re.error as e: + raise ValueError("Invalid commit message regex %r" % pattern) from e + # END handle invalid regex if rev is None: - commits = repo.iter_commits("--all") + commits = _all_ref_commits(repo) else: - commits = repo.iter_commits(to_commit(cast(Object, rev)).hexsha) + commits = _reachable_commits([to_commit(cast(Object, rev))]) # END handle starting point for commit in commits: @@ -499,6 +505,38 @@ def _find_commit_by_message( raise BadName("No commit found matching message pattern %r" % pattern) +def _all_ref_commits(repo: "Repo") -> Iterator["Commit"]: + starts = [] + for ref in repo.references: + try: + starts.append(to_commit(cast(Object, ref.object))) + except (BadName, ValueError): + pass + # END skip refs that do not point to commits + # END for each ref + try: + starts.append(repo.head.commit) + except ValueError: + pass + # END handle unborn head + return _reachable_commits(starts) + + +def _reachable_commits(starts: list["Commit"]) -> Iterator["Commit"]: + seen = set() + pending = starts[:] + while pending: + pending.sort(key=lambda commit: commit.committed_date, reverse=True) + commit = pending.pop(0) + if commit.binsha in seen: + continue + # END skip seen commit + seen.add(commit.binsha) + yield commit + pending.extend(commit.parents) + # END while commits remain + + def _index_lookup(repo: "Repo", spec: str) -> AnyGitObject: if not spec: raise ValueError("':' must be followed by a path") @@ -527,8 +565,6 @@ def _tree_lookup(obj: AnyGitObject, path: str) -> AnyGitObject: def _peel(obj: AnyGitObject, output_type: str, repo: "Repo", rev: str) -> AnyGitObject: - if output_type == "/": - return obj if output_type.startswith("/"): return _find_commit_by_message(repo, obj, output_type[1:], braced=True) if output_type == "": diff --git a/test/test_rev_parse.py b/test/test_rev_parse.py index d96fdc1a2..b00347668 100644 --- a/test/test_rev_parse.py +++ b/test/test_rev_parse.py @@ -1,8 +1,15 @@ +# Copyright (C) 2026 Michael Trier (mtrier@gmail.com) and contributors +# +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + from pathlib import Path import pytest from git import Repo +from git.refs import RemoteReference +from git.refs import SymbolicReference from gitdb.exc import BadName @@ -31,14 +38,12 @@ def rev_parse_repo(tmp_path): repo.create_tag("v1.0", ref=release) main = repo.active_branch - side = repo.create_head("side", root) - side.checkout() _write(repo, "side.txt", "side\n") - side_commit = repo.index.commit("side branch") + side_commit = repo.index.commit("side branch", parent_commits=[root], head=False, skip_hooks=True) + repo.create_head("side", side_commit) - main.checkout() - repo.git.merge("--no-ff", "side", "-m", "merge side") - merge = repo.head.commit + merge = repo.index.commit("merge side", parent_commits=[release, side_commit], skip_hooks=True) + repo.head.log_append(side_commit.binsha, "checkout: moving from side to main", merge.binsha) repo.create_head("aaaaaaaa", merge) repo.create_tag("@foo", ref=merge) @@ -55,16 +60,21 @@ def rev_parse_repo(tmp_path): def test_rev_parse_names_hex_and_describe_forms(rev_parse_repo): repo = rev_parse_repo["repo"] + release = rev_parse_repo["release"] merge = rev_parse_repo["merge"] assert repo.rev_parse("@") == merge assert repo.rev_parse("@foo") == merge assert repo.rev_parse("aaaaaaaa") == merge assert repo.rev_parse(merge.hexsha[:7]) == merge + describe_name = "anything-9-g%s" % merge.hexsha[:7] assert repo.rev_parse("v1.0-1-g%s" % merge.hexsha[:7]) == merge - assert repo.rev_parse("anything-9-g%s" % merge.hexsha[:7]) == merge + assert repo.rev_parse(describe_name) == merge assert repo.rev_parse("%s-dirty" % merge.hexsha[:7]) == merge + repo.create_tag(describe_name, ref=release) + assert repo.rev_parse(describe_name) == release + def test_rev_parse_navigation_and_peeling(rev_parse_repo): repo = rev_parse_repo["repo"] @@ -87,7 +97,8 @@ def test_rev_parse_navigation_and_peeling(rev_parse_repo): assert repo.rev_parse("ann^{}") == root assert repo.rev_parse("ann^{commit}") == root assert repo.rev_parse("HEAD^{tree}") == merge.tree - assert repo.rev_parse("HEAD^{/}") == merge + with pytest.raises(ValueError): + repo.rev_parse("HEAD^{/}") def test_rev_parse_tree_and_index_paths(rev_parse_repo): @@ -114,6 +125,10 @@ def test_rev_parse_reflog_selectors(rev_parse_repo): assert repo.rev_parse("%s@{0}" % main.name) == merge assert repo.rev_parse("@{-1}") == side + SymbolicReference.create(repo, "refs/remotes/origin/%s" % main.name, merge) + main.set_tracking_branch(RemoteReference(repo, "refs/remotes/origin/%s" % main.name)) + assert repo.rev_parse("%s@{upstream}" % main.name) == merge + def test_rev_parse_commit_message_search(rev_parse_repo): repo = rev_parse_repo["repo"] @@ -132,6 +147,10 @@ def test_rev_parse_rejects_invalid_object_specs(rev_parse_repo): repo.rev_parse(":") with pytest.raises(ValueError): repo.rev_parse(":/") + with pytest.raises(ValueError): + repo.rev_parse(":/[") + with pytest.raises(ValueError): + repo.rev_parse("HEAD^{/[}") with pytest.raises(ValueError): repo.rev_parse("@{-0}") with pytest.raises(ValueError): From aee2fd5c13770954469e650f1df8f92f0183bc70 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 29 Apr 2026 08:30:21 +0800 Subject: [PATCH 1312/1392] bump version to 3.1.49 --- VERSION | 2 +- doc/source/changes.rst | 11 +++++++++++ git/ext/gitdb | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 94c78f538..8335f2d61 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.48 +3.1.49 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 4ac67d077..020673826 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,17 @@ Changelog ========= +3.1.49 +====== + +Save setting of configuration values, +which cuold be used to inject other more configuration. + +Also more conforming `rev-parse` implementation. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.49 + 3.1.48 ====== diff --git a/git/ext/gitdb b/git/ext/gitdb index 5c1b3036a..335c0f661 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 5c1b3036a6e34782e0ab6ce85e5ae64fe777fdbe +Subproject commit 335c0f66173eecdc7b2597c2b6c3d1fde795df30 From 5e74292fb75f40e0c5dd883760ee764277e92b48 Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Thu, 30 Apr 2026 02:50:08 -0700 Subject: [PATCH 1313/1392] fix: replace deprecated codecs.open with built-in open (#128) Python 3.14 emits a DeprecationWarning for codecs.open(), which gitdb hits inside ReferenceDB._update_dbs_from_ref_file: DeprecationWarning: codecs.open() is deprecated. Use open() instead. The built-in open() has supported the encoding kwarg since Python 3.0 and the call site already passes encoding="utf-8", so the replacement is byte-for-byte equivalent on every supported Python version. Dropped the now-unused codecs import. Verified the change with the existing test_ref.py suite. Closes #128 --- gitdb/db/ref.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gitdb/db/ref.py b/gitdb/db/ref.py index bd3015602..5536db0f2 100644 --- a/gitdb/db/ref.py +++ b/gitdb/db/ref.py @@ -2,7 +2,6 @@ # # This module is part of GitDB and is released under # the New BSD License: https://opensource.org/license/bsd-3-clause/ -import codecs from gitdb.db.base import ( CompoundDB, ) @@ -42,7 +41,7 @@ def _update_dbs_from_ref_file(self): # try to get as many as possible, don't fail if some are unavailable ref_paths = list() try: - with codecs.open(self._ref_file, 'r', encoding="utf-8") as f: + with open(self._ref_file, 'r', encoding="utf-8") as f: ref_paths = [l.strip() for l in f] except OSError: pass From b17f11315b3c3baf7c073234670ce58cc2bbf5ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 May 2026 16:12:17 +0000 Subject: [PATCH 1314/1392] Bump https://github.com/astral-sh/ruff-pre-commit Bumps the pre-commit group with 1 update: [https://github.com/astral-sh/ruff-pre-commit](https://github.com/astral-sh/ruff-pre-commit). Updates `https://github.com/astral-sh/ruff-pre-commit` from v0.15.8 to 0.15.12 - [Release notes](https://github.com/astral-sh/ruff-pre-commit/releases) - [Commits](https://github.com/astral-sh/ruff-pre-commit/compare/v0.15.8...v0.15.12) --- updated-dependencies: - dependency-name: https://github.com/astral-sh/ruff-pre-commit dependency-version: 0.15.12 dependency-type: direct:production dependency-group: pre-commit ... Signed-off-by: dependabot[bot] --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 617111e1d..f3ab67035 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: exclude: ^test/fixtures/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.8 + rev: v0.15.12 hooks: - id: ruff-check args: ["--fix"] From 714e2e16dc2a67567ee48f7bcffcb59b9ca12caa Mon Sep 17 00:00:00 2001 From: "GPT 5.5" Date: Sun, 3 May 2026 00:02:49 +0800 Subject: [PATCH 1315/1392] Xfail Windows symlink-capable index mutation test The Windows CI jobs for PR 2140 failed in test/test_index.py::TestIndex::test_index_mutation. The failing checkout path creates my_fake_symlink and Git for Windows 2.54 reports a symlink warning before GitPython raises GitCommandError. This is the same unsupported Windows symlink behavior that the test already marks as an expected failure when core.symlinks is true. Detect Windows hosts that can create symlinks directly and include GitCommandError in the expected failure types, so symlink-capable Windows runners do not fail this unrelated Dependabot PR. Co-authored-by: Sebastian Thiel --- test/test_index.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/test/test_index.py b/test/test_index.py index 33490f907..f8280450a 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -172,6 +172,19 @@ def _decode(stdout): _win_bash_status = WinBashStatus.check() +def _windows_supports_symlinks(): + if sys.platform != "win32": + return False + + with tempfile.TemporaryDirectory(prefix="gitpython-symlink-check-") as temp_dir: + link_path = osp.join(temp_dir, "link") + try: + os.symlink("missing-target", link_path) + except (NotImplementedError, OSError): + return False + return S_ISLNK(os.lstat(link_path)[ST_MODE]) + + def _make_hook(git_dir, name, content, make_exec=True): """A helper to create a hook""" hp = hook_path(name, git_dir) @@ -553,9 +566,9 @@ def _count_existing(self, repo, files): # END num existing helper @pytest.mark.xfail( - sys.platform == "win32" and Git().config("core.symlinks") == "true", + sys.platform == "win32" and (Git().config("core.symlinks") == "true" or _windows_supports_symlinks()), reason="Assumes symlinks are not created on Windows and opens a symlink to a nonexistent target.", - raises=FileNotFoundError, + raises=(FileNotFoundError, GitCommandError), ) @with_rw_repo("0.1.6") def test_index_mutation(self, rw_repo): From 4e8cd45685d33c8b6af2f70c77a341c4a15acf14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 3 May 2026 13:32:18 +0000 Subject: [PATCH 1316/1392] Bump git/ext/gitdb from `335c0f6` to `53c94d6` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `335c0f6` to `53c94d6`. - [Release notes](https://github.com/gitpython-developers/gitdb/releases) - [Commits](https://github.com/gitpython-developers/gitdb/compare/335c0f66173eecdc7b2597c2b6c3d1fde795df30...53c94d682b541595918cea6fc2e96bb900eb0e8c) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-version: 53c94d682b541595918cea6fc2e96bb900eb0e8c dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 335c0f661..53c94d682 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 335c0f66173eecdc7b2597c2b6c3d1fde795df30 +Subproject commit 53c94d682b541595918cea6fc2e96bb900eb0e8c From 54538428f79b0c91ba52cda5229856a6edf7ac06 Mon Sep 17 00:00:00 2001 From: "GPT 5.5" Date: Tue, 5 May 2026 21:34:42 +0800 Subject: [PATCH 1317/1392] Validate config key section names before writing GitConfigParser already rejected CR, LF, and NUL in config values before writing, but section and option names could still reach configparser. Because GitPython writes section headers itself, a newline-bearing section name could split the output into additional headers. Reject CR, LF, and NUL in section and option names on write paths that create or set config keys: add_section(), set(), set_value(), add_value(), and rename_section() destinations. This matches Git config key validation behavior; Git source commit 94f057755b7941b321fd11fec1b2e3ca5313a4e0 reports invalid keys containing newlines from config.c, and local git 2.50.1 rejects newline-bearing config keys. Add a regression test covering unsafe section and option names while preserving safe writes. Co-authored-by: Sebastian Thiel --- git/config.py | 17 ++++++++++++++++- test/test_config.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/git/config.py b/git/config.py index 97ae054e5..82747eadd 100644 --- a/git/config.py +++ b/git/config.py @@ -72,6 +72,9 @@ See: https://git-scm.com/docs/git-config#_conditional_includes """ +UNSAFE_CONFIG_CHARS_RE = re.compile(r"[\r\n\x00]") +"""Characters that cannot be safely written in config names or values.""" + class MetaParserBuilder(abc.ABCMeta): # noqa: B024 """Utility class wrapping base-class methods into decorators that assure read-only @@ -778,6 +781,7 @@ def _assure_writable(self, method_name: str) -> None: def add_section(self, section: "cp._SectionName") -> None: """Assures added options will stay in order.""" + self._assure_config_name_safe(section, "section") return super().add_section(section) @property @@ -884,10 +888,14 @@ def _value_to_string(self, value: Union[str, bytes, int, float, bool]) -> str: def _value_to_string_safe(self, value: Union[str, bytes, int, float, bool]) -> str: value_str = self._value_to_string(value) - if re.search(r"[\r\n\x00]", value_str): + if UNSAFE_CONFIG_CHARS_RE.search(value_str): raise ValueError("Git config values must not contain CR, LF, or NUL") return value_str + def _assure_config_name_safe(self, name: "cp._SectionName", label: str) -> None: + if isinstance(name, str) and UNSAFE_CONFIG_CHARS_RE.search(name): + raise ValueError("Git config %s names must not contain CR, LF, or NUL" % label) + @needs_values @set_dirty_and_flush_changes def set( @@ -896,6 +904,8 @@ def set( option: str, value: Union[str, bytes, int, float, bool, None] = None, ) -> None: + self._assure_config_name_safe(section, "section") + self._assure_config_name_safe(option, "option") if value is not None: value = self._value_to_string_safe(value) return super().set(section, option, value) @@ -920,6 +930,8 @@ def set_value(self, section: str, option: str, value: Union[str, bytes, int, flo :return: This instance """ + self._assure_config_name_safe(section, "section") + self._assure_config_name_safe(option, "option") value_str = self._value_to_string_safe(value) if not self.has_section(section): self.add_section(section) @@ -948,6 +960,8 @@ def add_value(self, section: str, option: str, value: Union[str, bytes, int, flo :return: This instance """ + self._assure_config_name_safe(section, "section") + self._assure_config_name_safe(option, "option") value_str = self._value_to_string_safe(value) if not self.has_section(section): self.add_section(section) @@ -968,6 +982,7 @@ def rename_section(self, section: str, new_name: str) -> "GitConfigParser": """ if not self.has_section(section): raise ValueError("Source section '%s' doesn't exist" % section) + self._assure_config_name_safe(new_name, "section") if self.has_section(new_name): raise ValueError("Destination section '%s' already exists" % new_name) diff --git a/test/test_config.py b/test/test_config.py index a9dcdb087..3ddaf0a4b 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -163,6 +163,37 @@ def test_set_value_rejects_config_injection(self, rw_dir): self.assertFalse(git_config.has_section("user")) self.assertFalse(git_config.has_section("core")) + @with_rw_directory + def test_set_value_rejects_unsafe_section_and_option_names(self, rw_dir): + config_path = osp.join(rw_dir, "config") + bad_keys = ("user]\n[core", "user]\r[core", "user]\x00[core") + + with GitConfigParser(config_path, read_only=False) as git_config: + git_config.add_section("user") + for bad_key in bad_keys: + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.add_section(bad_key) + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set(bad_key, "hooksPath", "/tmp/hooks") + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set("user", bad_key, "/tmp/hooks") + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set_value(bad_key, "hooksPath", "/tmp/hooks") + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.set_value("user", bad_key, "/tmp/hooks") + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.add_value(bad_key, "hooksPath", "/tmp/hooks") + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.add_value("user", bad_key, "/tmp/hooks") + with pytest.raises(ValueError, match="CR, LF, or NUL"): + git_config.rename_section("user", bad_key) + + git_config.set_value("user", "name", "safe") + + with GitConfigParser(config_path, read_only=True) as git_config: + self.assertEqual(git_config.get_value("user", "name"), "safe") + self.assertFalse(git_config.has_section("core")) + @with_rw_directory def test_set_and_add_value_reject_unsafe_value_characters(self, rw_dir): config_path = osp.join(rw_dir, "config") From 5a294a6fc7ed5dc0946d4b576257bf926178f269 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 6 May 2026 12:00:26 +0800 Subject: [PATCH 1318/1392] bump version to 3.1.50 --- VERSION | 2 +- doc/source/changes.rst | 10 +++++++++- git/ext/gitdb | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 8335f2d61..0bc461141 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.49 +3.1.50 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 020673826..b5152b3c5 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,11 +2,19 @@ Changelog ========= +3.1.50 +====== + +Save setting of configuration values, this time sections as well, as follow-up to 3.1.49. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.50 + 3.1.49 ====== Save setting of configuration values, -which cuold be used to inject other more configuration. +which could be used to inject other more configuration. Also more conforming `rev-parse` implementation. diff --git a/git/ext/gitdb b/git/ext/gitdb index 53c94d682..335c0f661 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 53c94d682b541595918cea6fc2e96bb900eb0e8c +Subproject commit 335c0f66173eecdc7b2597c2b6c3d1fde795df30 From 4941c314fc8c38ed6df037c29d4809f7a7fd9af9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 6 May 2026 14:52:31 +0800 Subject: [PATCH 1319/1392] Add AI-disclusure and quality requirements to the contribution guidelines. Co-authored-by: GPT 5.5 --- CONTRIBUTING.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8536d7f73..d8a29f55d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,35 @@ The following is a short step-by-step rundown of what one typically would do to - Feel free to add yourself to AUTHORS file. - Create a pull request. +## Quality expectations + +Contributions must be made with care and meet the quality bar of the surrounding code. +That means a change should not leave GitPython worse than it was before: it should be +readable, maintainable, tested where practical, documented and consistent with the +existing style and behavior. +A contribution that works only narrowly but lowers the quality of the +codebase may be declined, and the pull request closed without warning. + +## AI-assisted contributions + +If AI edits files for you, disclose it in the pull request description and commit +metadata. Prefer making the agent identity part of the commit, for example by using +an AI author such as `$agent $version ` or a co-author via +a `Co-authored-by: ` trailer. + +Agents operating through a person's GitHub account must identify themselves. For +example, comments posted by an agent should say so directly with phrases like +`AI agent on behalf of : ...`. + +Fully AI-generated comments on pull requests or issues must also be disclosed. +Undisclosed AI-generated comments may lead to the pull request or issue being closed. + +AI-assisted proofreading or wording polish does not need disclosure, but it is still +courteous to mention it when the AI materially influenced the final text. + +Automated or "full-auto" AI contributions without a human responsible for reviewing +and standing behind the work may be closed. + ## Fuzzing Test Specific Documentation For details related to contributing to the fuzzing test suite and OSS-Fuzz integration, please From 92ff6df86b82e94254874b52b05dee9ffe38716b Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 6 May 2026 09:58:33 -0400 Subject: [PATCH 1320/1392] Separate quality paragraphs and adjust decline wording Split the quality-expectations section into two paragraphs (the warning about low-quality contributions being declined was visually merged with the preceding paragraph). Replace "and the pull request closed without warning" with a note that maintainers may not always be able to provide detailed feedback, which conveys the same practical reality. Co-authored-by: Claude Opus 4.6 --- CONTRIBUTING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d8a29f55d..60e34a651 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,8 +15,10 @@ Contributions must be made with care and meet the quality bar of the surrounding That means a change should not leave GitPython worse than it was before: it should be readable, maintainable, tested where practical, documented and consistent with the existing style and behavior. + A contribution that works only narrowly but lowers the quality of the -codebase may be declined, and the pull request closed without warning. +codebase may be declined. The maintainers may not always be able to provide +detailed feedback. ## AI-assisted contributions From d172e541455679ebdfb55df1411836298cb47b8d Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Thu, 7 May 2026 03:20:20 -0700 Subject: [PATCH 1321/1392] docs(cmd): clarify Git.execute() string vs list command argument Closes #2016 Users routinely hit GitCommandNotFound by passing a single string with spaces to repo.git.execute(...). With shell=False (default) subprocess treats the entire string as the executable name and fails. Document the recommended list form, the string-as-single-executable behavior, and the two ways to coerce a string into argv tokens (shlex.split or shell=True). --- git/cmd.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 78a9f4c78..1ecff9318 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1119,9 +1119,16 @@ def execute( information (stdout). :param command: - The command argument list to execute. - It should be a sequence of program arguments, or a string. The - program to execute is the first item in the args sequence or string. + The command to execute. A sequence of program arguments is the + recommended form when `shell` is ``False`` (the default), e.g. + ``["git", "log", "-n", "1"]``. + + A string is accepted, but with `shell` set to ``False`` it is passed + as a single executable name to :class:`subprocess.Popen`. For example, + ``"git log -n 1"`` looks for an executable literally named + ``git log -n 1`` and will fail with :class:`GitCommandNotFound`. To + split a command string into argv tokens, pass ``shlex.split(...)`` as + a sequence or set `shell` to ``True`` (see the warning below). :param istream: Standard input filehandle passed to :class:`subprocess.Popen`. From 81cf3e5fd2a19842b21f30e158b7f41bf6d91f28 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Fri, 8 May 2026 04:38:09 -0700 Subject: [PATCH 1322/1392] Fix DecompressMemMapReader.read returning b'' before EOF Closes #120 DecompressMemMapReader.read(N) could return b'' mid-stream when zlib consumed input without producing output on a single decompress call (small N, header / dictionary frames in flight). The original `if dcompdat and ...` guard at the recursion site skipped the "refill to size" recursion in that case, so a caller using the standard idiom while chunk := stream.read(4096): yield chunk terminated at the first empty chunk -- before _br == _s. The guard exists for compressed_bytes_read(), which manipulates _br=0 and then drains the inner zip past its EOF. Recursing there would loop forever because the inner zip is already done. The fix uses zlib's own `eof` attribute (available on standard zlib.Decompress objects since Python 3.6) to distinguish: - dcompdat empty AND zip not at EOF -> still digesting, recurse - dcompdat empty AND zip at EOF -> compressed_bytes_read scrub or genuine EOF; do not recurse. `getattr(_zip, 'eof', False)` keeps the conservative behavior when running against a custom zlib object that does not expose the attribute. Adds a regression test that reads with chunk_size in {1, 4, 16, 64} from a 13 KB highly-compressible stream. With the old guard, the chunk_size <= 16 cases stopped at byte 0; the new test asserts they read all 13000 bytes. The full existing test suite (24 tests) still passes, including test_decompress_reader_special_case and test_pack which exercise the compressed_bytes_read scrub path that the original guard existed to protect. --- gitdb/stream.py | 136 ++++++++++++++++++++++---------------- gitdb/test/test_stream.py | 35 ++++++++++ 2 files changed, 114 insertions(+), 57 deletions(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index 1e0be84fd..f5fc3bc57 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -254,68 +254,90 @@ def read(self, size=-1): # copied once, and another copy of a part of it when it creates the unconsumed # tail. We have to use it to hand in the appropriate amount of bytes during # the next read. - tail = self._zip.unconsumed_tail - if tail: - # move the window, make it as large as size demands. For code-clarity, - # we just take the chunk from our map again instead of reusing the unconsumed - # tail. The latter one would safe some memory copying, but we could end up - # with not getting enough data uncompressed, so we had to sort that out as well. - # Now we just assume the worst case, hence the data is uncompressed and the window - # needs to be as large as the uncompressed bytes we want to read. - self._cws = self._cwe - len(tail) - self._cwe = self._cws + size - else: - cws = self._cws - self._cws = self._cwe - self._cwe = cws + size - # END handle tail - - # if window is too small, make it larger so zip can decompress something - if self._cwe - self._cws < 8: - self._cwe = self._cws + 8 - # END adjust winsize - - # takes a slice, but doesn't copy the data, it says ... - indata = self._m[self._cws:self._cwe] - - # get the actual window end to be sure we don't use it for computations - self._cwe = self._cws + len(indata) - dcompdat = self._zip.decompress(indata, size) - # update the amount of compressed bytes read - # We feed possibly overlapping chunks, which is why the unconsumed tail - # has to be taken into consideration, as well as the unused data - # if we hit the end of the stream - # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. - # They are thorough, and I assume it is truly working. - # Why is this logic as convoluted as it is ? Please look at the table in - # https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. - # Basically, on py2.6, you want to use branch 1, whereas on all other python version, the second branch - # will be the one that works. - # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the - # table in the github issue. This is it ... it was the only way I could make this work everywhere. - # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . - if getattr(zlib, 'ZLIB_RUNTIME_VERSION', zlib.ZLIB_VERSION) in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': - unused_datalen = len(self._zip.unconsumed_tail) - else: - unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) - # # end handle very special case ... - - self._cbr += len(indata) - unused_datalen - self._br += len(dcompdat) + # + # Decompress in a loop until we have produced `size` bytes or run out + # of progress. Iteration (instead of recursion) keeps the call bounded + # for streams that consume many input bytes per produced output byte + # (e.g. zlib stored blocks of length zero); the previous recursive + # form blew the stack on inputs > ~1500 empty blocks (issue #120 + # follow-up). + dcompdat = b'' + while True: + tail = self._zip.unconsumed_tail + remaining = size - len(dcompdat) + if tail: + # move the window, make it as large as size demands. For code-clarity, + # we just take the chunk from our map again instead of reusing the unconsumed + # tail. The latter one would safe some memory copying, but we could end up + # with not getting enough data uncompressed, so we had to sort that out as well. + # Now we just assume the worst case, hence the data is uncompressed and the window + # needs to be as large as the uncompressed bytes we want to read. + self._cws = self._cwe - len(tail) + self._cwe = self._cws + remaining + else: + cws = self._cws + self._cws = self._cwe + self._cwe = cws + remaining + # END handle tail + + # if window is too small, make it larger so zip can decompress something + if self._cwe - self._cws < 8: + self._cwe = self._cws + 8 + # END adjust winsize + + # takes a slice, but doesn't copy the data, it says ... + indata = self._m[self._cws:self._cwe] + + # get the actual window end to be sure we don't use it for computations + self._cwe = self._cws + len(indata) + chunk = self._zip.decompress(indata, remaining) + # update the amount of compressed bytes read + # We feed possibly overlapping chunks, which is why the unconsumed tail + # has to be taken into consideration, as well as the unused data + # if we hit the end of the stream + # NOTE: Behavior changed in PY2.7 onward, which requires special handling to make the tests work properly. + # They are thorough, and I assume it is truly working. + # Why is this logic as convoluted as it is ? Please look at the table in + # https://github.com/gitpython-developers/gitdb/issues/19 to learn about the test-results. + # Basically, on py2.6, you want to use branch 1, whereas on all other python version, the second branch + # will be the one that works. + # However, the zlib VERSIONs as well as the platform check is used to further match the entries in the + # table in the github issue. This is it ... it was the only way I could make this work everywhere. + # IT's CERTAINLY GOING TO BITE US IN THE FUTURE ... . + if getattr(zlib, 'ZLIB_RUNTIME_VERSION', zlib.ZLIB_VERSION) in ('1.2.7', '1.2.5') and not sys.platform == 'darwin': + unused_datalen = len(self._zip.unconsumed_tail) + else: + unused_datalen = len(self._zip.unconsumed_tail) + len(self._zip.unused_data) + # # end handle very special case ... + + consumed = len(indata) - unused_datalen + self._cbr += consumed + self._br += len(chunk) + dcompdat += chunk + + # Stop when we have enough or there is no path to more output. + # `chunk` may legitimately be empty mid-stream when zlib is + # consuming header / dictionary frames; in that case we keep + # iterating as long as we are still feeding zlib new bytes + # (consumed > 0) and zlib has not flagged end-of-stream. The + # compressed_bytes_read() scrub loop drives this same code with + # _br manipulated to 0 past zip EOF; it terminates here because + # `getattr(_zip, 'eof', False)` is True or no compressed bytes + # are consumed. The empty-block recursion attack from issue #120 + # follow-up is bounded by the iteration; each empty block does + # consume input, so the loop walks the stream forward a constant + # amount per iteration without growing the call stack. + if len(dcompdat) >= size or self._br >= self._s: + break + zip_eof = getattr(self._zip, 'eof', False) + if not chunk and (zip_eof or len(indata) == 0 or consumed == 0): + break + # END iterative decompress if dat: dcompdat = dat + dcompdat # END prepend our cached data - # it can happen, depending on the compression, that we get less bytes - # than ordered as it needs the final portion of the data as well. - # Recursively resolve that. - # Note: dcompdat can be empty even though we still appear to have bytes - # to read, if we are called by compressed_bytes_read - it manipulates - # us to empty the stream - if dcompdat and (len(dcompdat) - len(dat)) < size and self._br < self._s: - dcompdat += self.read(size - len(dcompdat)) - # END handle special case return dcompdat diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 1e7e941d1..390caa182 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -162,3 +162,38 @@ def test_decompress_reader_special_case(self): dump = mdb.store(IStream(ostream.type, ostream.size, BytesIO(data))) assert dump.hexsha == sha # end for each loose object sha to test + + def test_decompress_reader_chunked_read_does_not_terminate_early(self): + """Regression test for #120: read(N) must not return b'' before EOF. + + zlib can consume input without producing decompressed output (e.g. + while ingesting block headers). The reader's internal recursion + previously bailed on any empty zip output, so a caller reading in + small chunks via the standard `while chunk := stream.read(N)` idiom + would terminate at the first empty chunk -- before the actual end + of the uncompressed stream. + """ + # Highly compressible data exposes the bug because each zlib chunk + # spans many uncompressed bytes -- intermediate decompress() calls + # often return empty while consuming input. + data = b"hello world! " * 1000 + zdata = zlib.compress(data) + + # Loop with a small chunk size to force many sub-_s recursions. + for chunk_size in (1, 4, 16, 64): + reader = DecompressMemMapReader( + zdata, close_on_deletion=False, size=len(data) + ) + out = bytearray() + while True: + chunk = reader.read(chunk_size) + if not chunk: + break + out.extend(chunk) + assert bytes(out) == data, ( + f"chunk_size={chunk_size}: got {len(out)}/{len(data)} bytes" + ) + assert reader._br == reader._s, ( + f"chunk_size={chunk_size}: stream stopped at " + f"{reader._br}/{reader._s}" + ) From ed41d2c26f77bc58a5b8f51100a300c4e09a7a30 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 9 May 2026 08:29:18 +0800 Subject: [PATCH 1323/1392] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- gitdb/stream.py | 7 +++++-- gitdb/test/test_stream.py | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/gitdb/stream.py b/gitdb/stream.py index f5fc3bc57..7c59f0b3c 100644 --- a/gitdb/stream.py +++ b/gitdb/stream.py @@ -268,7 +268,7 @@ def read(self, size=-1): if tail: # move the window, make it as large as size demands. For code-clarity, # we just take the chunk from our map again instead of reusing the unconsumed - # tail. The latter one would safe some memory copying, but we could end up + # tail. The latter one would save some memory copying, but we could end up # with not getting enough data uncompressed, so we had to sort that out as well. # Now we just assume the worst case, hence the data is uncompressed and the window # needs to be as large as the uncompressed bytes we want to read. @@ -313,7 +313,10 @@ def read(self, size=-1): consumed = len(indata) - unused_datalen self._cbr += consumed self._br += len(chunk) - dcompdat += chunk + if chunk: + if not isinstance(dcompdat, bytearray): + dcompdat = bytearray(dcompdat) + dcompdat.extend(chunk) # Stop when we have enough or there is no path to more output. # `chunk` may legitimately be empty mid-stream when zlib is diff --git a/gitdb/test/test_stream.py b/gitdb/test/test_stream.py index 390caa182..f36b06b96 100644 --- a/gitdb/test/test_stream.py +++ b/gitdb/test/test_stream.py @@ -179,7 +179,8 @@ def test_decompress_reader_chunked_read_does_not_terminate_early(self): data = b"hello world! " * 1000 zdata = zlib.compress(data) - # Loop with a small chunk size to force many sub-_s recursions. + # Loop with a small chunk size to force many internal read/decompression + # iterations before EOF. for chunk_size in (1, 4, 16, 64): reader = DecompressMemMapReader( zdata, close_on_deletion=False, size=len(data) From 010a7bbfd237d2ab0627c20a542933abf978b690 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Fri, 8 May 2026 16:34:31 -0400 Subject: [PATCH 1324/1392] Rewrite Git.execute() command parameter docstring per #2146 #2146 identifies two factual problems in #2144's `:param command:` rewrite, both of which this fixes: 1. The claim that with `shell=False` a string "is passed as a single executable name to `subprocess.Popen`" is accurate on Unix-like systems, but on Windows `subprocess.Popen` forwards the string to `CreateProcessW` and Windows command-line parsing produces argv. So multi-word strings happen to work on Windows -- which makes the docstring misleading for Windows readers and unhelpful for anyone whose actual problem is portability. 2. The blanket recommendation to use `shlex.split` read as a general string-to-argv splitter. `shlex.split` parses POSIX shell syntax on all systems, and the resulting tokens are still unsafe for anything but fixed, fully trusted strings -- in particular, strings built by interpolating values can still inject extra arguments via embedded whitespace or quoting. Restructure `:param command:` into four paragraphs (description, platform-specific string handling, `shell=True` / `Git.USE_SHELL` warning, qualified `shlex.split` note) and cross-reference `USE_SHELL` for the long-form security discussion rather than reproducing it. Add a related paragraph to `:param shell:` noting `shlex.split`'s narrow value as a transitional tool when migrating existing string-command `shell=True` calls on Unix-like systems, with extreme care, and cross-referencing `:param command:` for the risks. This fixes #2146. Only documentation is changed. Co-authored-by: Claude Opus 4.7 (1M context) --- git/cmd.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 7f2564d45..861009099 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1131,16 +1131,28 @@ def execute( information (stdout). :param command: - The command to execute. A sequence of program arguments is the - recommended form when `shell` is ``False`` (the default), e.g. - ``["git", "log", "-n", "1"]``. - - A string is accepted, but with `shell` set to ``False`` it is passed - as a single executable name to :class:`subprocess.Popen`. For example, - ``"git log -n 1"`` looks for an executable literally named - ``git log -n 1`` and will fail with :class:`GitCommandNotFound`. To - split a command string into argv tokens, pass ``shlex.split(...)`` as - a sequence or set `shell` to ``True`` (see the warning below). + The command to execute. A sequence of program arguments is recommended. + A string is also accepted, but its meaning is strongly platform-dependent. + + By default, a shell is not used. On Unix-like systems, a string is the whole + program name (so ``"git log -n 1"`` raises :class:`GitCommandNotFound`). On + Windows, the program parses the arguments itself, so multi-word strings can + work but are not portable. + + Avoid ``shell=True`` (and :attr:`Git.USE_SHELL`): this runs the command in + a shell, which is generally unsafe. The shell interprets metacharacters + such as ``;``, ``|``, ``&``, ``$(...)``, ``$VAR``, ``%VAR%``, and ``^`` + (depending on the platform) as syntax. Any untrusted text in the command + can then execute arbitrary OS commands. See :attr:`Git.USE_SHELL`. + + Producing a sequence automatically by :func:`shlex.split` and passing it + as the command is far safer than ``shell=True``. But :func:`shlex.split` + parses POSIX shell syntax on all systems, and the result is still unsafe + for anything but *fixed, fully trusted* strings. Do not use it on strings + built by interpolating values: whitespace or quoting in an untrusted value + can still inject arguments. For input derived in any way from untrusted + data, build the argument sequence yourself, while ensuring each argument + is fully sanitized. :param istream: Standard input filehandle passed to :class:`subprocess.Popen`. @@ -1208,6 +1220,11 @@ def execute( needed (nor useful) to work around any known operating system specific issues. + On Unix-like systems, when migrating away from passing string commands with + ``shell=True``, :func:`shlex.split` may serve as a transitional step in rare + cases, but this should be undertaken with extreme care. See the `command` + parameter above on the risks. + :param env: A dictionary of environment variables to be passed to :class:`subprocess.Popen`. From 38d62c4b60b5cd2ed743c00f2cac1875c76449e0 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 May 2026 21:19:10 -0400 Subject: [PATCH 1325/1392] Word `shell` mention of `shlex.split` more carefully To avert the mistake of not removing `shell=True` when using it. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 (1M context) --- git/cmd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 861009099..98bb62775 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1222,8 +1222,8 @@ def execute( On Unix-like systems, when migrating away from passing string commands with ``shell=True``, :func:`shlex.split` may serve as a transitional step in rare - cases, but this should be undertaken with extreme care. See the `command` - parameter above on the risks. + cases, with extreme care. (Drop ``shell=True`` and pass the resulting + sequence as the command.) See the `command` parameter above on the risks. :param env: A dictionary of environment variables to be passed to From 0c1e40982438b46e2e47dc833bccdced779a47db Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 9 May 2026 21:49:47 -0400 Subject: [PATCH 1326/1392] Document init script behavior when multiple remotes have master When `master` is locally absent and more than one remote has it, `git checkout master --` fails by default even if all remotes agree, and the script falls back to creating `master` at `HEAD`. The reflog populated by the subsequent resets then traces `HEAD`'s history rather than a remote `master`'s. This is harmless, because `master` is reset to `__testing_point__` either way, but unintuitive. Add a comment so a reader of the script does not have to discover this from a confusing run. This fixes #2145. Co-Authored-By: Claude Opus 4.7 (1M context) --- init-tests-after-clone.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index bfada01b0..a88f983fc 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -40,6 +40,11 @@ fi git tag __testing_point__ # The tests need a branch called master. +# +# If master is locally absent but more than one remote has it, checkout fails +# by default even if all remotes agree, and we fall back to creating it at +# HEAD. The reflog we populate below then traces HEAD's history rather than +# a remote master's, but master is reset to __testing_point__ either way. git checkout master -- || git checkout -b master # The tests need a reflog history on the master branch. From 465f1d587a13795b35a4b72ad7fafbbe9c7731ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 May 2026 13:32:12 +0000 Subject: [PATCH 1327/1392] Bump git/ext/gitdb from `335c0f6` to `0a019a2` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `335c0f6` to `0a019a2`. - [Release notes](https://github.com/gitpython-developers/gitdb/releases) - [Commits](https://github.com/gitpython-developers/gitdb/compare/335c0f66173eecdc7b2597c2b6c3d1fde795df30...0a019a2e2bd73158cf8b637ad78b5d4b8f15e42e) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-version: 0a019a2e2bd73158cf8b637ad78b5d4b8f15e42e dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 335c0f661..0a019a2e2 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 335c0f66173eecdc7b2597c2b6c3d1fde795df30 +Subproject commit 0a019a2e2bd73158cf8b637ad78b5d4b8f15e42e From c334167a41f63fe6a16090d894d3bb26332bee9b Mon Sep 17 00:00:00 2001 From: "E-Love (Eric Loveland)" Date: Thu, 14 May 2026 21:57:51 -0400 Subject: [PATCH 1328/1392] fix(worktree): align with Git's gitdir resolution Git only normalizes relative references in `gitdir`[1], so do the same. [1]: https://github.com/git/git/blob/v2.54.0/setup.c#L1012-L1025 --- git/repo/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/repo/base.py b/git/repo/base.py index 7579e326f..08707fc5c 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -295,7 +295,7 @@ def __init__( sm_gitpath = find_worktree_git_dir(dotgit) if sm_gitpath is not None: - git_dir = expand_path(sm_gitpath, expand_vars) + git_dir = osp.normpath(sm_gitpath) self._working_tree_dir = curpath break From e94ccaff11ca7f4388e920bcc2ff617dd9e2a827 Mon Sep 17 00:00:00 2001 From: "E-Love (Eric Loveland)" Date: Tue, 12 May 2026 15:49:14 -0400 Subject: [PATCH 1329/1392] support relative worktree paths (git 2.48+ worktree.useRelativePaths) git 2.48 introduced the `worktree.useRelativePaths` config option, which causes `git worktree add` to write a relative `gitdir` into the worktree's `.git` file. Resolve the relative `gitdir` against the worktree directory before normalizing it. Absolute paths are unaffected because `os.path.join` ignores the prefix when joined with an absolute path. --- git/repo/base.py | 3 ++- test/test_repo.py | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/git/repo/base.py b/git/repo/base.py index 08707fc5c..2d3cf24f0 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -295,7 +295,8 @@ def __init__( sm_gitpath = find_worktree_git_dir(dotgit) if sm_gitpath is not None: - git_dir = osp.normpath(sm_gitpath) + # worktrees can use relative paths as of Git 2.48, so we join to curpath + git_dir = osp.normpath(osp.join(curpath, sm_gitpath)) self._working_tree_dir = curpath break diff --git a/test/test_repo.py b/test/test_repo.py index fae3dc0b9..13bff52e9 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -1094,7 +1094,7 @@ def test_is_valid_object(self): self.assertFalse(repo.is_valid_object(tag_sha, "commit")) @with_rw_directory - def test_git_work_tree_dotgit(self, rw_dir): + def test_git_work_tree_dotgit(self, rw_dir, use_relative_paths=False): """Check that we find .git as a worktree file and find the worktree based on it.""" git = Git(rw_dir) @@ -1106,7 +1106,11 @@ def test_git_work_tree_dotgit(self, rw_dir): worktree_path = join_path_native(rw_dir, "worktree_repo") if Git.is_cygwin(): worktree_path = cygpath(worktree_path) - rw_master.git.worktree("add", worktree_path, branch.name) + wt_add_kwargs = {"insert_kwargs_after": "add"} + # relative worktree paths introduced in git 2.48.0 + if use_relative_paths and git.version_info[:3] >= (2, 48, 0): + wt_add_kwargs["relative_paths"] = True + rw_master.git.worktree("add", worktree_path, branch.name, **wt_add_kwargs) # This ensures that we can read the repo's gitdir correctly. repo = Repo(worktree_path) @@ -1124,6 +1128,15 @@ def test_git_work_tree_dotgit(self, rw_dir): self.assertIsInstance(repo.heads["aaaaaaaa"], Head) + def test_git_work_tree_dotgit_relative(self): + """Check that we find .git as a worktree file containing a relative path + and find the worktree based on it.""" + if Git().version_info[:3] < (2, 48, 0): + pytest.skip("relative worktree feature unsupported, needs git 2.48.0 or later") + # this class inherits from TestCase so we can't use pytest.mark.parametrize on + # test_git_work_tree_dotgit; delegate instead + self.test_git_work_tree_dotgit(use_relative_paths=True) + @with_rw_directory def test_git_work_tree_env(self, rw_dir): """Check that we yield to GIT_WORK_TREE.""" From 29c98613fd7f286786cf450ffa8a2e314f4317e7 Mon Sep 17 00:00:00 2001 From: "E-Love (Eric Loveland)" Date: Tue, 12 May 2026 16:16:44 -0400 Subject: [PATCH 1330/1392] doc(_call_process): correct example --- git/cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index 7f2564d45..637225f0b 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1595,7 +1595,7 @@ def _call_process( turns into:: - git rev-list --max-count=10 --header=master + git rev-list --max-count=10 --header master :return: Same as :meth:`execute`. If no args are given, used :meth:`execute`'s From 03baced04b00f4505201e38b0507fb8d071e29ca Mon Sep 17 00:00:00 2001 From: "E-Love (Eric Loveland)" Date: Fri, 15 May 2026 11:51:11 -0400 Subject: [PATCH 1331/1392] Add xfail_if_raises context manager to defer xfail condition evaluation The @pytest.mark.xfail decorator with raises=... evaluates its condition during test collection, which can trigger external calls (e.g., Git().config()) that have side effects. Introduce xfail_if_raises as a context manager that delays this evaluation until test execution time, and apply it to test_index_mutation. --- test/lib/helper.py | 27 ++ test/test_index.py | 721 +++++++++++++++++++++++---------------------- 2 files changed, 388 insertions(+), 360 deletions(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index 6a8b714e6..2fc015dfa 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -18,6 +18,7 @@ "skipIf", "GIT_REPO", "GIT_DAEMON_PORT", + "xfail_if_raises", ] import contextlib @@ -35,8 +36,10 @@ import time import unittest import venv +from typing import Union, Type, Tuple import gitdb +import pytest from git.util import rmtree, cwd @@ -465,3 +468,27 @@ def _executable(self, basename): if osp.isfile(path) or osp.islink(path): return path raise RuntimeError(f"no regular file or symlink {path!r}") + + +@contextlib.contextmanager +def xfail_if_raises( + condition: bool, + *, + raises: Union[Type[BaseException], Tuple[Type[BaseException], ...]], + reason: str = "", + strict: bool = False, +): + """Approximates the behavior of @pytest.mark.xfail(..., raises=...) as a context + manager that can be used within a test, such as when the condition is complex or has + side effects + + One difference is it will not report XPASS if the test passes, but setting `strict` + simulates it by raising an exception""" + try: + yield + except raises: + if condition: + pytest.xfail(reason) + raise + if strict and condition: + pytest.fail("[XPASS(strict)] " + reason) diff --git a/test/test_index.py b/test/test_index.py index f8280450a..cb45d3e90 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -38,6 +38,7 @@ from git.util import Actor, cwd, hex_to_bin, rmtree from test.lib import TestBase, VirtualEnvironment, fixture, fixture_path, with_rw_directory, with_rw_repo, PathLikeMock +from test.lib.helper import xfail_if_raises HOOKS_SHEBANG = "#!/usr/bin/env sh\n" @@ -565,369 +566,369 @@ def _count_existing(self, repo, files): # END num existing helper - @pytest.mark.xfail( - sys.platform == "win32" and (Git().config("core.symlinks") == "true" or _windows_supports_symlinks()), - reason="Assumes symlinks are not created on Windows and opens a symlink to a nonexistent target.", - raises=(FileNotFoundError, GitCommandError), - ) @with_rw_repo("0.1.6") def test_index_mutation(self, rw_repo): - index = rw_repo.index - num_entries = len(index.entries) - cur_head = rw_repo.head - - uname = "Thomas Müller" - umail = "sd@company.com" - with rw_repo.config_writer() as writer: - writer.set_value("user", "name", uname) - writer.set_value("user", "email", umail) - self.assertEqual(writer.get_value("user", "name"), uname) - - # Remove all of the files, provide a wild mix of paths, BaseIndexEntries, - # IndexEntries. - def mixed_iterator(): - count = 0 - for entry in index.entries.values(): - type_id = count % 5 - if type_id == 0: # path (str) - yield entry.path - elif type_id == 1: # path (PathLike) - yield Path(entry.path) - elif type_id == 2: # path mock (PathLike) - yield PathLikeMock(entry.path) - elif type_id == 3: # path mock in a blob - yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) - elif type_id == 4: # blob - yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) - elif type_id == 5: # BaseIndexEntry - yield BaseIndexEntry(entry[:4]) - elif type_id == 6: # IndexEntry - yield entry - else: - raise AssertionError("Invalid Type") - count += 1 - # END for each entry - - # END mixed iterator - deleted_files = index.remove(mixed_iterator(), working_tree=False) - assert deleted_files - self.assertEqual(self._count_existing(rw_repo, deleted_files), len(deleted_files)) - self.assertEqual(len(index.entries), 0) - - # Reset the index to undo our changes. - index.reset() - self.assertEqual(len(index.entries), num_entries) - - # Remove with working copy. - deleted_files = index.remove(mixed_iterator(), working_tree=True) - assert deleted_files - self.assertEqual(self._count_existing(rw_repo, deleted_files), 0) - - # Reset everything. - index.reset(working_tree=True) - self.assertEqual(self._count_existing(rw_repo, deleted_files), len(deleted_files)) - - # Invalid type. - self.assertRaises(TypeError, index.remove, [1]) - - # Absolute path. - deleted_files = index.remove([osp.join(rw_repo.working_tree_dir, "lib")], r=True) - assert len(deleted_files) > 1 - self.assertRaises(ValueError, index.remove, ["/doesnt/exists"]) - - # TEST COMMITTING - # Commit changed index. - cur_commit = cur_head.commit - commit_message = "commit default head by Frèderic Çaufl€" - - new_commit = index.commit(commit_message, head=False) - assert cur_commit != new_commit - self.assertEqual(new_commit.author.name, uname) - self.assertEqual(new_commit.author.email, umail) - self.assertEqual(new_commit.committer.name, uname) - self.assertEqual(new_commit.committer.email, umail) - self.assertEqual(new_commit.message, commit_message) - self.assertEqual(new_commit.parents[0], cur_commit) - self.assertEqual(len(new_commit.parents), 1) - self.assertEqual(cur_head.commit, cur_commit) - - # Commit with other actor. - cur_commit = cur_head.commit - - my_author = Actor("Frèderic Çaufl€", "author@example.com") - my_committer = Actor("Committing Frèderic Çaufl€", "committer@example.com") - commit_actor = index.commit(commit_message, author=my_author, committer=my_committer) - assert cur_commit != commit_actor - self.assertEqual(commit_actor.author.name, "Frèderic Çaufl€") - self.assertEqual(commit_actor.author.email, "author@example.com") - self.assertEqual(commit_actor.committer.name, "Committing Frèderic Çaufl€") - self.assertEqual(commit_actor.committer.email, "committer@example.com") - self.assertEqual(commit_actor.message, commit_message) - self.assertEqual(commit_actor.parents[0], cur_commit) - self.assertEqual(len(new_commit.parents), 1) - self.assertEqual(cur_head.commit, commit_actor) - self.assertEqual(cur_head.log()[-1].actor, my_committer) - - # Commit with author_date and commit_date. - cur_commit = cur_head.commit - commit_message = "commit with dates by Avinash Sajjanshetty" - - new_commit = index.commit( - commit_message, - author_date="2006-04-07T22:13:13", - commit_date="2005-04-07T22:13:13", - ) - assert cur_commit != new_commit - print(new_commit.authored_date, new_commit.committed_date) - self.assertEqual(new_commit.message, commit_message) - self.assertEqual(new_commit.authored_date, 1144447993) - self.assertEqual(new_commit.committed_date, 1112911993) - - # Same index, no parents. - commit_message = "index without parents" - commit_no_parents = index.commit(commit_message, parent_commits=[], head=True) - self.assertEqual(commit_no_parents.message, commit_message) - self.assertEqual(len(commit_no_parents.parents), 0) - self.assertEqual(cur_head.commit, commit_no_parents) - - # same index, multiple parents. - commit_message = "Index with multiple parents\n commit with another line" - commit_multi_parent = index.commit(commit_message, parent_commits=(commit_no_parents, new_commit)) - self.assertEqual(commit_multi_parent.message, commit_message) - self.assertEqual(len(commit_multi_parent.parents), 2) - self.assertEqual(commit_multi_parent.parents[0], commit_no_parents) - self.assertEqual(commit_multi_parent.parents[1], new_commit) - self.assertEqual(cur_head.commit, commit_multi_parent) - - # Re-add all files in lib. - # Get the lib folder back on disk, but get an index without it. - index.reset(new_commit.parents[0], working_tree=True).reset(new_commit, working_tree=False) - lib_file_path = osp.join("lib", "git", "__init__.py") - assert (lib_file_path, 0) not in index.entries - assert osp.isfile(osp.join(rw_repo.working_tree_dir, lib_file_path)) - - # Directory. - entries = index.add(["lib"], fprogress=self._fprogress_add) - self._assert_entries(entries) - self._assert_fprogress(entries) - assert len(entries) > 1 - - # Glob. - entries = index.reset(new_commit).add([osp.join("lib", "git", "*.py")], fprogress=self._fprogress_add) - self._assert_entries(entries) - self._assert_fprogress(entries) - self.assertEqual(len(entries), 14) - - # Same file. - entries = index.reset(new_commit).add( - [osp.join(rw_repo.working_tree_dir, "lib", "git", "head.py")] * 2, - fprogress=self._fprogress_add, - ) - self._assert_entries(entries) - self.assertEqual(entries[0].mode & 0o644, 0o644) - # Would fail, test is too primitive to handle this case. - # self._assert_fprogress(entries) - self._reset_progress() - self.assertEqual(len(entries), 2) - - # Missing path. - self.assertRaises(OSError, index.reset(new_commit).add, ["doesnt/exist/must/raise"]) - - # Blob from older revision overrides current index revision. - old_blob = new_commit.parents[0].tree.blobs[0] - entries = index.reset(new_commit).add([old_blob], fprogress=self._fprogress_add) - self._assert_entries(entries) - self._assert_fprogress(entries) - self.assertEqual(index.entries[(old_blob.path, 0)].hexsha, old_blob.hexsha) - self.assertEqual(len(entries), 1) - - # Mode 0 not allowed. - null_hex_sha = Diff.NULL_HEX_SHA - null_bin_sha = b"\0" * 20 - self.assertRaises( - ValueError, - index.reset(new_commit).add, - [BaseIndexEntry((0, null_bin_sha, 0, "doesntmatter"))], - ) - - # Add new file. - new_file_relapath = "my_new_file" - self._make_file(new_file_relapath, "hello world", rw_repo) - entries = index.reset(new_commit).add( - [BaseIndexEntry((0o10644, null_bin_sha, 0, new_file_relapath))], - fprogress=self._fprogress_add, - ) - self._assert_entries(entries) - self._assert_fprogress(entries) - self.assertEqual(len(entries), 1) - self.assertNotEqual(entries[0].hexsha, null_hex_sha) - - # Add symlink. - if sys.platform != "win32": - for target in ("/etc/nonexisting", "/etc/passwd", "/etc"): - basename = "my_real_symlink" - - link_file = osp.join(rw_repo.working_tree_dir, basename) - os.symlink(target, link_file) - entries = index.reset(new_commit).add([link_file], fprogress=self._fprogress_add) - self._assert_entries(entries) - self._assert_fprogress(entries) - self.assertEqual(len(entries), 1) - self.assertTrue(S_ISLNK(entries[0].mode)) - self.assertTrue(S_ISLNK(index.entries[index.entry_key("my_real_symlink", 0)].mode)) - - # We expect only the target to be written. - self.assertEqual( - index.repo.odb.stream(entries[0].binsha).read().decode("ascii"), - target, - ) - - os.remove(link_file) - # END for each target - # END real symlink test - - # Add fake symlink and assure it checks out as a symlink. - fake_symlink_relapath = "my_fake_symlink" - link_target = "/etc/that" - fake_symlink_path = self._make_file(fake_symlink_relapath, link_target, rw_repo) - fake_entry = BaseIndexEntry((0o120000, null_bin_sha, 0, fake_symlink_relapath)) - entries = index.reset(new_commit).add([fake_entry], fprogress=self._fprogress_add) - self._assert_entries(entries) - self._assert_fprogress(entries) - assert entries[0].hexsha != null_hex_sha - self.assertEqual(len(entries), 1) - self.assertTrue(S_ISLNK(entries[0].mode)) - - # Check that this also works with an alternate method. - full_index_entry = IndexEntry.from_base(BaseIndexEntry((0o120000, entries[0].binsha, 0, entries[0].path))) - entry_key = index.entry_key(full_index_entry) - index.reset(new_commit) - - assert entry_key not in index.entries - index.entries[entry_key] = full_index_entry - index.write() - index.update() # Force reread of entries. - new_entry = index.entries[entry_key] - assert S_ISLNK(new_entry.mode) + with xfail_if_raises( + sys.platform == "win32" and (Git().config("core.symlinks") == "true" or _windows_supports_symlinks()), + raises=(FileNotFoundError, GitCommandError), + reason="Assumes symlinks are not created on Windows and opens a symlink to a nonexistent target.", + ): + index = rw_repo.index + num_entries = len(index.entries) + cur_head = rw_repo.head + + uname = "Thomas Müller" + umail = "sd@company.com" + with rw_repo.config_writer() as writer: + writer.set_value("user", "name", uname) + writer.set_value("user", "email", umail) + self.assertEqual(writer.get_value("user", "name"), uname) + + # Remove all of the files, provide a wild mix of paths, BaseIndexEntries, + # IndexEntries. + def mixed_iterator(): + count = 0 + for entry in index.entries.values(): + type_id = count % 5 + if type_id == 0: # path (str) + yield entry.path + elif type_id == 1: # path (PathLike) + yield Path(entry.path) + elif type_id == 2: # path mock (PathLike) + yield PathLikeMock(entry.path) + elif type_id == 3: # path mock in a blob + yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) + elif type_id == 4: # blob + yield Blob(rw_repo, entry.binsha, entry.mode, entry.path) + elif type_id == 5: # BaseIndexEntry + yield BaseIndexEntry(entry[:4]) + elif type_id == 6: # IndexEntry + yield entry + else: + raise AssertionError("Invalid Type") + count += 1 + # END for each entry + + # END mixed iterator + deleted_files = index.remove(mixed_iterator(), working_tree=False) + assert deleted_files + self.assertEqual(self._count_existing(rw_repo, deleted_files), len(deleted_files)) + self.assertEqual(len(index.entries), 0) + + # Reset the index to undo our changes. + index.reset() + self.assertEqual(len(index.entries), num_entries) + + # Remove with working copy. + deleted_files = index.remove(mixed_iterator(), working_tree=True) + assert deleted_files + self.assertEqual(self._count_existing(rw_repo, deleted_files), 0) + + # Reset everything. + index.reset(working_tree=True) + self.assertEqual(self._count_existing(rw_repo, deleted_files), len(deleted_files)) + + # Invalid type. + self.assertRaises(TypeError, index.remove, [1]) + + # Absolute path. + deleted_files = index.remove([osp.join(rw_repo.working_tree_dir, "lib")], r=True) + assert len(deleted_files) > 1 + self.assertRaises(ValueError, index.remove, ["/doesnt/exists"]) + + # TEST COMMITTING + # Commit changed index. + cur_commit = cur_head.commit + commit_message = "commit default head by Frèderic Çaufl€" + + new_commit = index.commit(commit_message, head=False) + assert cur_commit != new_commit + self.assertEqual(new_commit.author.name, uname) + self.assertEqual(new_commit.author.email, umail) + self.assertEqual(new_commit.committer.name, uname) + self.assertEqual(new_commit.committer.email, umail) + self.assertEqual(new_commit.message, commit_message) + self.assertEqual(new_commit.parents[0], cur_commit) + self.assertEqual(len(new_commit.parents), 1) + self.assertEqual(cur_head.commit, cur_commit) + + # Commit with other actor. + cur_commit = cur_head.commit + + my_author = Actor("Frèderic Çaufl€", "author@example.com") + my_committer = Actor("Committing Frèderic Çaufl€", "committer@example.com") + commit_actor = index.commit(commit_message, author=my_author, committer=my_committer) + assert cur_commit != commit_actor + self.assertEqual(commit_actor.author.name, "Frèderic Çaufl€") + self.assertEqual(commit_actor.author.email, "author@example.com") + self.assertEqual(commit_actor.committer.name, "Committing Frèderic Çaufl€") + self.assertEqual(commit_actor.committer.email, "committer@example.com") + self.assertEqual(commit_actor.message, commit_message) + self.assertEqual(commit_actor.parents[0], cur_commit) + self.assertEqual(len(new_commit.parents), 1) + self.assertEqual(cur_head.commit, commit_actor) + self.assertEqual(cur_head.log()[-1].actor, my_committer) + + # Commit with author_date and commit_date. + cur_commit = cur_head.commit + commit_message = "commit with dates by Avinash Sajjanshetty" + + new_commit = index.commit( + commit_message, + author_date="2006-04-07T22:13:13", + commit_date="2005-04-07T22:13:13", + ) + assert cur_commit != new_commit + print(new_commit.authored_date, new_commit.committed_date) + self.assertEqual(new_commit.message, commit_message) + self.assertEqual(new_commit.authored_date, 1144447993) + self.assertEqual(new_commit.committed_date, 1112911993) + + # Same index, no parents. + commit_message = "index without parents" + commit_no_parents = index.commit(commit_message, parent_commits=[], head=True) + self.assertEqual(commit_no_parents.message, commit_message) + self.assertEqual(len(commit_no_parents.parents), 0) + self.assertEqual(cur_head.commit, commit_no_parents) + + # same index, multiple parents. + commit_message = "Index with multiple parents\n commit with another line" + commit_multi_parent = index.commit(commit_message, parent_commits=(commit_no_parents, new_commit)) + self.assertEqual(commit_multi_parent.message, commit_message) + self.assertEqual(len(commit_multi_parent.parents), 2) + self.assertEqual(commit_multi_parent.parents[0], commit_no_parents) + self.assertEqual(commit_multi_parent.parents[1], new_commit) + self.assertEqual(cur_head.commit, commit_multi_parent) + + # Re-add all files in lib. + # Get the lib folder back on disk, but get an index without it. + index.reset(new_commit.parents[0], working_tree=True).reset(new_commit, working_tree=False) + lib_file_path = osp.join("lib", "git", "__init__.py") + assert (lib_file_path, 0) not in index.entries + assert osp.isfile(osp.join(rw_repo.working_tree_dir, lib_file_path)) + + # Directory. + entries = index.add(["lib"], fprogress=self._fprogress_add) + self._assert_entries(entries) + self._assert_fprogress(entries) + assert len(entries) > 1 + + # Glob. + entries = index.reset(new_commit).add([osp.join("lib", "git", "*.py")], fprogress=self._fprogress_add) + self._assert_entries(entries) + self._assert_fprogress(entries) + self.assertEqual(len(entries), 14) + + # Same file. + entries = index.reset(new_commit).add( + [osp.join(rw_repo.working_tree_dir, "lib", "git", "head.py")] * 2, + fprogress=self._fprogress_add, + ) + self._assert_entries(entries) + self.assertEqual(entries[0].mode & 0o644, 0o644) + # Would fail, test is too primitive to handle this case. + # self._assert_fprogress(entries) + self._reset_progress() + self.assertEqual(len(entries), 2) + + # Missing path. + self.assertRaises(OSError, index.reset(new_commit).add, ["doesnt/exist/must/raise"]) + + # Blob from older revision overrides current index revision. + old_blob = new_commit.parents[0].tree.blobs[0] + entries = index.reset(new_commit).add([old_blob], fprogress=self._fprogress_add) + self._assert_entries(entries) + self._assert_fprogress(entries) + self.assertEqual(index.entries[(old_blob.path, 0)].hexsha, old_blob.hexsha) + self.assertEqual(len(entries), 1) + + # Mode 0 not allowed. + null_hex_sha = Diff.NULL_HEX_SHA + null_bin_sha = b"\0" * 20 + self.assertRaises( + ValueError, + index.reset(new_commit).add, + [BaseIndexEntry((0, null_bin_sha, 0, "doesntmatter"))], + ) - # A tree created from this should contain the symlink. - tree = index.write_tree() - assert fake_symlink_relapath in tree - index.write() # Flush our changes for the checkout. - - # Check out the fake link, should be a link then. - assert not S_ISLNK(os.stat(fake_symlink_path)[ST_MODE]) - os.remove(fake_symlink_path) - index.checkout(fake_symlink_path) - - # On Windows, we currently assume we will never get symlinks. - if sys.platform == "win32": - # Symlinks should contain the link as text (which is what a - # symlink actually is). - with open(fake_symlink_path, "rt") as fd: - self.assertEqual(fd.read(), link_target) - else: - self.assertTrue(S_ISLNK(os.lstat(fake_symlink_path)[ST_MODE])) - - # TEST RENAMING - def assert_mv_rval(rval): - for source, dest in rval: - assert not osp.exists(source) and osp.exists(dest) - # END for each renamed item - - # END move assertion utility - - self.assertRaises(ValueError, index.move, ["just_one_path"]) - # Try to move a file onto an existing file. - files = ["AUTHORS", "LICENSE"] - self.assertRaises(GitCommandError, index.move, files) - - # Again, with force. - assert_mv_rval(index.move(files, f=True)) - - # Move files into a directory - dry run. - paths = ["LICENSE", "VERSION", "doc"] - rval = index.move(paths, dry_run=True) - self.assertEqual(len(rval), 2) - assert osp.exists(paths[0]) - - # Again, no dry run. - rval = index.move(paths) - assert_mv_rval(rval) - - # Move dir into dir. - rval = index.move(["doc", "test"]) - assert_mv_rval(rval) - - # TEST PATH REWRITING - ###################### - count = [0] - - def rewriter(entry): - rval = str(count[0]) - count[0] += 1 - return rval - - # END rewriter - - def make_paths(): - """Help out the test by yielding two existing paths and one new path.""" - yield "CHANGES" - yield "ez_setup.py" - yield index.entries[index.entry_key("README", 0)] - yield index.entries[index.entry_key(".gitignore", 0)] - - for fid in range(3): - fname = "newfile%i" % fid - with open(fname, "wb") as fd: - fd.write(b"abcd") - yield Blob(rw_repo, Blob.NULL_BIN_SHA, 0o100644, fname) - # END for each new file - - # END path producer - paths = list(make_paths()) - self._assert_entries(index.add(paths, path_rewriter=rewriter)) - - for filenum in range(len(paths)): - assert index.entry_key(str(filenum), 0) in index.entries - - # TEST RESET ON PATHS - ###################### - arela = "aa" - brela = "bb" - afile = self._make_file(arela, "adata", rw_repo) - bfile = self._make_file(brela, "bdata", rw_repo) - akey = index.entry_key(arela, 0) - bkey = index.entry_key(brela, 0) - keys = (akey, bkey) - absfiles = (afile, bfile) - files = (arela, brela) - - for fkey in keys: - assert fkey not in index.entries - - index.add(files, write=True) - nc = index.commit("2 files committed", head=False) - - for fkey in keys: - assert fkey in index.entries - - # Just the index. - index.reset(paths=(arela, afile)) - assert akey not in index.entries - assert bkey in index.entries - - # Now with working tree - files on disk as well as entries must be recreated. - rw_repo.head.commit = nc - for absfile in absfiles: - os.remove(absfile) - - index.reset(working_tree=True, paths=files) - - for fkey in keys: - assert fkey in index.entries - for absfile in absfiles: - assert osp.isfile(absfile) + # Add new file. + new_file_relapath = "my_new_file" + self._make_file(new_file_relapath, "hello world", rw_repo) + entries = index.reset(new_commit).add( + [BaseIndexEntry((0o10644, null_bin_sha, 0, new_file_relapath))], + fprogress=self._fprogress_add, + ) + self._assert_entries(entries) + self._assert_fprogress(entries) + self.assertEqual(len(entries), 1) + self.assertNotEqual(entries[0].hexsha, null_hex_sha) + + # Add symlink. + if sys.platform != "win32": + for target in ("/etc/nonexisting", "/etc/passwd", "/etc"): + basename = "my_real_symlink" + + link_file = osp.join(rw_repo.working_tree_dir, basename) + os.symlink(target, link_file) + entries = index.reset(new_commit).add([link_file], fprogress=self._fprogress_add) + self._assert_entries(entries) + self._assert_fprogress(entries) + self.assertEqual(len(entries), 1) + self.assertTrue(S_ISLNK(entries[0].mode)) + self.assertTrue(S_ISLNK(index.entries[index.entry_key("my_real_symlink", 0)].mode)) + + # We expect only the target to be written. + self.assertEqual( + index.repo.odb.stream(entries[0].binsha).read().decode("ascii"), + target, + ) + + os.remove(link_file) + # END for each target + # END real symlink test + + # Add fake symlink and assure it checks out as a symlink. + fake_symlink_relapath = "my_fake_symlink" + link_target = "/etc/that" + fake_symlink_path = self._make_file(fake_symlink_relapath, link_target, rw_repo) + fake_entry = BaseIndexEntry((0o120000, null_bin_sha, 0, fake_symlink_relapath)) + entries = index.reset(new_commit).add([fake_entry], fprogress=self._fprogress_add) + self._assert_entries(entries) + self._assert_fprogress(entries) + assert entries[0].hexsha != null_hex_sha + self.assertEqual(len(entries), 1) + self.assertTrue(S_ISLNK(entries[0].mode)) + + # Check that this also works with an alternate method. + full_index_entry = IndexEntry.from_base(BaseIndexEntry((0o120000, entries[0].binsha, 0, entries[0].path))) + entry_key = index.entry_key(full_index_entry) + index.reset(new_commit) + + assert entry_key not in index.entries + index.entries[entry_key] = full_index_entry + index.write() + index.update() # Force reread of entries. + new_entry = index.entries[entry_key] + assert S_ISLNK(new_entry.mode) + + # A tree created from this should contain the symlink. + tree = index.write_tree() + assert fake_symlink_relapath in tree + index.write() # Flush our changes for the checkout. + + # Check out the fake link, should be a link then. + assert not S_ISLNK(os.stat(fake_symlink_path)[ST_MODE]) + os.remove(fake_symlink_path) + index.checkout(fake_symlink_path) + + # On Windows, we currently assume we will never get symlinks. + if sys.platform == "win32": + # Symlinks should contain the link as text (which is what a + # symlink actually is). + with open(fake_symlink_path, "rt") as fd: + self.assertEqual(fd.read(), link_target) + else: + self.assertTrue(S_ISLNK(os.lstat(fake_symlink_path)[ST_MODE])) + + # TEST RENAMING + def assert_mv_rval(rval): + for source, dest in rval: + assert not osp.exists(source) and osp.exists(dest) + # END for each renamed item + + # END move assertion utility + + self.assertRaises(ValueError, index.move, ["just_one_path"]) + # Try to move a file onto an existing file. + files = ["AUTHORS", "LICENSE"] + self.assertRaises(GitCommandError, index.move, files) + + # Again, with force. + assert_mv_rval(index.move(files, f=True)) + + # Move files into a directory - dry run. + paths = ["LICENSE", "VERSION", "doc"] + rval = index.move(paths, dry_run=True) + self.assertEqual(len(rval), 2) + assert osp.exists(paths[0]) + + # Again, no dry run. + rval = index.move(paths) + assert_mv_rval(rval) + + # Move dir into dir. + rval = index.move(["doc", "test"]) + assert_mv_rval(rval) + + # TEST PATH REWRITING + ###################### + count = [0] + + def rewriter(entry): + rval = str(count[0]) + count[0] += 1 + return rval + + # END rewriter + + def make_paths(): + """Help out the test by yielding two existing paths and one new path.""" + yield "CHANGES" + yield "ez_setup.py" + yield index.entries[index.entry_key("README", 0)] + yield index.entries[index.entry_key(".gitignore", 0)] + + for fid in range(3): + fname = "newfile%i" % fid + with open(fname, "wb") as fd: + fd.write(b"abcd") + yield Blob(rw_repo, Blob.NULL_BIN_SHA, 0o100644, fname) + # END for each new file + + # END path producer + paths = list(make_paths()) + self._assert_entries(index.add(paths, path_rewriter=rewriter)) + + for filenum in range(len(paths)): + assert index.entry_key(str(filenum), 0) in index.entries + + # TEST RESET ON PATHS + ###################### + arela = "aa" + brela = "bb" + afile = self._make_file(arela, "adata", rw_repo) + bfile = self._make_file(brela, "bdata", rw_repo) + akey = index.entry_key(arela, 0) + bkey = index.entry_key(brela, 0) + keys = (akey, bkey) + absfiles = (afile, bfile) + files = (arela, brela) + + for fkey in keys: + assert fkey not in index.entries + + index.add(files, write=True) + nc = index.commit("2 files committed", head=False) + + for fkey in keys: + assert fkey in index.entries + + # Just the index. + index.reset(paths=(arela, afile)) + assert akey not in index.entries + assert bkey in index.entries + + # Now with working tree - files on disk as well as entries must be recreated. + rw_repo.head.commit = nc + for absfile in absfiles: + os.remove(absfile) + + index.reset(working_tree=True, paths=files) + + for fkey in keys: + assert fkey in index.entries + for absfile in absfiles: + assert osp.isfile(absfile) @with_rw_repo("HEAD") def test_compare_write_tree(self, rw_repo): From fa0d9bc17928e7c11dc11c875c1047587f5354c3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 May 2026 11:35:08 -0400 Subject: [PATCH 1332/1392] Add a test that fixture directories are usable by git Many tests rely on git accepting their fixture directories: the GitPython repository itself, and the gitdb and smmap submodules used by the test suite as direct and nested submodule fixtures. When git rejects one for "dubious ownership" (typically because a CI workflow's `safe.directory` list is missing an entry), three submodule-related tests fail in opaque ways. The exact failure type depends on which side of a race wins in the flush to the cached `git cat-file --batch-check` subprocess: - When Python wins, `ValueError` reaches the test directly. - When git wins, `BrokenPipeError` is caught internally as `IOError` and `iter_items` returns early. Each test then fails on the empty-or-short result in its own way: - `test_docs::Tutorials::test_submodules` indexes the empty `sm.children()` list with `[0]` and raises `IndexError`. - `test_repo::TestRepo::test_submodules` and `test_submodule::TestSubmodule::test_root_module` see a length-1 (not length-2) submodule list from their recursive traversals and fail their length assertion with `AssertionError`. This commit adds `test/test_fixture_health.py`, which runs `git rev-parse --show-toplevel` in each fixture directory and asserts both that git is willing to operate there and that it reports the directory as its own toplevel. The failure message names `safe.directory` and ownership as the likely causes, so a misconfigured environment is recognizable directly from the test output rather than needing to be pieced together from a cascade of failing tests not conceptually related to safe directory protections. This adds the test only. Further replication and a fix are forthcoming. On the Cygwin CI workflow now, the test fails for `[gitdb]` and passes for `[repo_root]` and `[smmap]`. The asymmetry between the two submodules relates to NTFS ownership inherited from `actions/checkout`'s clone of the main tree. Co-authored-by: Claude Opus 4.7 (1M context) --- test/test_fixture_health.py | 91 +++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 test/test_fixture_health.py diff --git a/test/test_fixture_health.py b/test/test_fixture_health.py new file mode 100644 index 000000000..b2e8cfa6b --- /dev/null +++ b/test/test_fixture_health.py @@ -0,0 +1,91 @@ +# This module is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +"""Verify that fixture directories are usable by git. + +If a directory the test suite relies on is rejected by git for "dubious +ownership" -- because the directory's owner doesn't match the running user +and there is no ``safe.directory`` entry overriding the check -- three +submodule-related tests fail in confusing ways. The checks here name the +root cause clearly so a misconfigured environment is recognizable from the +test output. + +The rejection is most often a CI-workflow problem (the workflow's +``safe.directory`` list doesn't cover the path); on a developer's own +clone, it usually reflects an ownership mismatch (sudo clone, restored +backup, container mount, networked filesystem) rather than a config gap. + +These tests do not exercise GitPython's production code. They verify the +conditions under which production code is exercised are valid. + +A check is skipped, rather than failed, if a fixture directory is missing or +has no ``.git`` marker, since that condition is more naturally diagnosed as +"``init-tests-after-clone.sh`` hasn't been run" than as a problem with +``safe.directory``. +""" + +import subprocess +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# Directories git must trust for the test suite to operate normally. The +# current set is the GitPython working tree plus the working trees of its +# gitdb submodule and the smmap submodule nested inside gitdb. New entries +# should be added here whenever the test suite gains a dependency on git +# accepting another directory. +FIXTURE_DIRS = [ + pytest.param(REPO_ROOT, id="repo_root"), + pytest.param(REPO_ROOT / "git" / "ext" / "gitdb", id="gitdb"), + pytest.param( + REPO_ROOT / "git" / "ext" / "gitdb" / "gitdb" / "ext" / "smmap", + id="smmap", + ), +] + + +@pytest.mark.parametrize("fixture_dir", FIXTURE_DIRS) +def test_fixture_dir_is_trusted_by_git(fixture_dir: Path) -> None: + """git accepts ``fixture_dir`` as its own repository owned by a trusted user. + + Run ``git -C rev-parse --show-toplevel`` and assert it + succeeds and reports ``fixture_dir`` itself as the toplevel. Failure + typically means the directory's on-disk ownership doesn't match the + running user and the CI workflow's ``safe.directory`` list is missing + an entry that would override the check. + """ + if not fixture_dir.exists(): + pytest.skip(f"{fixture_dir} not present (run `git submodule update --init --recursive` from the repo root)") + if not (fixture_dir / ".git").exists(): + pytest.skip( + f"{fixture_dir} has no .git marker " + "(submodule not initialized; run " + "`git submodule update --init --recursive` from the repo root)" + ) + try: + result = subprocess.run( + ["git", "-C", str(fixture_dir), "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=False, + ) + except FileNotFoundError: + pytest.skip("git is not installed or not on PATH") + assert result.returncode == 0, ( + f"git refuses to operate in {fixture_dir}.\n" + f"stderr: {result.stderr.strip()}\n" + "The directory's owner doesn't match the running user and no " + "`safe.directory` entry overrides the check. On CI, the " + "workflow's `safe.directory` list typically needs an entry for " + "this path. Locally, this is unexpected and usually indicates " + "an ownership problem worth investigating." + ) + reported = Path(result.stdout.strip()) + assert reported.samefile(fixture_dir), ( + f"git reports the toplevel as {reported}, " + f"not as {fixture_dir} itself. " + "This usually means the directory is not an initialized git " + "repository (its `.git` marker may be stale or pointing elsewhere)." + ) From d5fb280613feb71a335ae371db1c32d3de09d594 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Thu, 7 May 2026 18:55:39 -0400 Subject: [PATCH 1333/1392] Add a test that required submodules are initialized This adds the new test, `test_required_submodule_is_initialized`, to `test_fixture_health.py`. For the gitdb and smmap submodules, the test asserts that the working tree directory exists and contains a `.git` marker (file or directory). The failure message reminds the developer to run `git submodule update --init --recursive`. This is a regression test for a different contract than the preexisting `test_fixture_dir_is_trusted_by_git` in `test_fixture_health.py`. That test verifies git's willingness to operate in each fixture directory (the `safe.directory` / dubious-ownership contract). This one verifies that the directories are populated at all. When the gitdb and smmap submodules aren't populated, the rest of the suite fails in ways that don't name the cause: a cascade of failures across `test_docs.py`, `test_repo.py`, and `test_submodule.py` -- tests that need submodule content but don't set out to check for it. That cascade was the failure mode of #1713 on the Arch Linux build, where `init-tests-after-clone.sh` had stopped running `git submodule update` on CI. Wherever the submodules are actually missing, this test reports the missing path directly instead. The test skips when the source tree is not a git clone (no `.git` at `REPO_ROOT`). This is to accommodate setups that run pytest from an extracted release tarball and prepare submodules in a separately-pointed tree via `GIT_PYTHON_TEST_GIT_REPO_BASE` (e.g. OpenIndiana's package build). In such setups, `git submodule update` cannot operate on the source tree, and the submodule paths checked here would never be populated there, but the test suite still works because `rorepo` points elsewhere. Co-authored-by: Claude Opus 4.7 (1M context) --- test/test_fixture_health.py | 76 ++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/test/test_fixture_health.py b/test/test_fixture_health.py index b2e8cfa6b..b18d5e8f9 100644 --- a/test/test_fixture_health.py +++ b/test/test_fixture_health.py @@ -3,25 +3,15 @@ """Verify that fixture directories are usable by git. -If a directory the test suite relies on is rejected by git for "dubious -ownership" -- because the directory's owner doesn't match the running user -and there is no ``safe.directory`` entry overriding the check -- three -submodule-related tests fail in confusing ways. The checks here name the -root cause clearly so a misconfigured environment is recognizable from the -test output. +If a fixture directory is missing, isn't an initialized git repository, +or is rejected by git for "dubious ownership", dependent tests +elsewhere in the suite fail in opaque ways. The checks here name the +preconditions directly so a misconfigured environment is recognizable +from the test output rather than from a cascade of unrelated-seeming +failures. -The rejection is most often a CI-workflow problem (the workflow's -``safe.directory`` list doesn't cover the path); on a developer's own -clone, it usually reflects an ownership mismatch (sudo clone, restored -backup, container mount, networked filesystem) rather than a config gap. - -These tests do not exercise GitPython's production code. They verify the -conditions under which production code is exercised are valid. - -A check is skipped, rather than failed, if a fixture directory is missing or -has no ``.git`` marker, since that condition is more naturally diagnosed as -"``init-tests-after-clone.sh`` hasn't been run" than as a problem with -``safe.directory``. +These tests do not exercise GitPython's production code. They verify +the conditions under which production code is exercised are valid. """ import subprocess @@ -45,6 +35,19 @@ ), ] +# Submodule working trees that must be present and initialized for the +# test suite to operate normally: gitdb at `git/ext/gitdb`, and smmap +# nested inside gitdb at `git/ext/gitdb/gitdb/ext/smmap`. The paths +# below are anchored at REPO_ROOT (the GitPython source tree), not at +# any rorepo redirection target. +SUBMODULE_DIRS = [ + pytest.param(REPO_ROOT / "git" / "ext" / "gitdb", id="gitdb"), + pytest.param( + REPO_ROOT / "git" / "ext" / "gitdb" / "gitdb" / "ext" / "smmap", + id="smmap", + ), +] + @pytest.mark.parametrize("fixture_dir", FIXTURE_DIRS) def test_fixture_dir_is_trusted_by_git(fixture_dir: Path) -> None: @@ -89,3 +92,40 @@ def test_fixture_dir_is_trusted_by_git(fixture_dir: Path) -> None: "This usually means the directory is not an initialized git " "repository (its `.git` marker may be stale or pointing elsewhere)." ) + + +@pytest.mark.parametrize("submodule_dir", SUBMODULE_DIRS) +def test_required_submodule_is_initialized(submodule_dir: Path) -> None: + """The submodule's working tree is present and initialized. + + Failure means the source tree is a git clone but the submodule's + working tree hasn't been populated. Skipped when the source tree + itself isn't a git clone (e.g. an extracted release tarball), since + ``git submodule update`` cannot operate there; setups that handle + submodules in a separately-prepared tree (via + ``GIT_PYTHON_TEST_GIT_REPO_BASE``) are exempted from this check. + """ + if not (REPO_ROOT / ".git").exists(): + pytest.skip( + "Source tree is not a git clone (no .git in REPO_ROOT); submodules " + "cannot be initialized via `git submodule update` here. Setups " + "that prepare submodules in a separately-pointed tree (via " + "GIT_PYTHON_TEST_GIT_REPO_BASE) are exempted from this check." + ) + # The assertion messages below recommend `git submodule update --init + # --recursive` rather than `init-tests-after-clone.sh`, even though the + # latter is the documented entry point for first-time test setup. Two + # reasons: the script performs `git reset --hard` operations that can + # destroy local work, and #1713 showed the script itself can carry + # submodule-init regressions, in which case recommending it would lead + # developers in a circle. The direct git command is a safe minimal fix + # for this test's specific failure mode and bypasses any such regression. + assert submodule_dir.is_dir(), ( + f"Submodule working tree missing: {submodule_dir}.\n" + "Run `git submodule update --init --recursive` from the repo root." + ) + assert (submodule_dir / ".git").exists(), ( + f"Submodule directory exists but has no .git marker: {submodule_dir}.\n" + "The submodule hasn't been initialized. " + "Run `git submodule update --init --recursive` from the repo root." + ) From ce68322cc3b3fea4077b3a1800283d00fe4dcb3e Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 6 May 2026 14:26:57 -0400 Subject: [PATCH 1334/1392] Clearly reproduce Cygwin safe.directory submodule bug Remove the Cygwin xfail decorations from `test_submodules` in `test_docs.py` and `test_repo.py`, and from `test_root_module` in `test_submodule.py`, so the tests surface the underlying failure directly. Also temporarily add 256 `reproduce-safe-dir` matrix jobs to `.github/workflows/cygwin-test.yml`, each running these three tests under the current `safe.directory` configuration. All or most of these `reproduce-safe-dir` jobs shall be removed before the associated bugfix is integrated. The existing test job's `env`, `defaults`, and setup steps gain YAML anchors so the new job can reference them without duplication. The anchors will be removed when the `reproduce-safe-dir` jobs are. Root cause ---------- The CI workflow adds the main repo and its `.git` directory to `safe.directory` but not the gitdb or smmap submodule working trees. Git matches `safe.directory` by specific paths (or wildcards, but we are not using wildcards). When GitPython opens a submodule as a `Repo` and runs `git cat-file --batch-check` against it, git rejects the repository for dubious ownership and exits. The race -------- GitPython caches the cat-file process. When git has exited, the next call to `__get_object_header` hits one of two outcomes depending on whether Python's `cmd.stdin.flush()` runs before or after the kernel marks the read end of the pipe as closed. (This is the same mechanism as the long-standing #427, in which SIGINT kills the cat-file process and the next `flush()` raises `BrokenPipeError`.) 1. If Python wins the race, `flush()` succeeds (data goes into the kernel buffer); the subsequent `stdout.readline()` returns `b""` (EOF); `_parse_object_header` raises a `ValueError` whose message names the rejected directory. 2. If git wins, `flush()` raises `BrokenPipeError`, a subclass of `OSError` (a.k.a. `IOError`). In both paths, the exception travels from `Object.new_from_sha` (outside `name_to_object`'s `except ValueError`, which covers only the `dereference_recursive` call), up through `repo.commit("HEAD")`, into `iter_items` in `git/objects/submodule/base.py`. That function's `except (IOError, BadName)` clause catches `BrokenPipeError` but not `ValueError`. So path (1) propagates `ValueError` all the way to the test, while path (2) ends with `iter_items` returning early and the test seeing an empty submodule list. Per-test outcomes ----------------- What the test sees in path (2) is determined by what the test does with the empty list: - `test_docs::test_submodules` does `sm.children()[0].name`, so the `[0]` on the empty list raises `IndexError`. - `test_repo::test_submodules` does `assertGreaterEqual(len(list(self.rorepo.iter_submodules())), 2)`. The recursive traversal yields gitdb (its `iter_items` on the *main* repo succeeds, because the main repo IS in `safe.directory`) but not smmap, so length 1 fails the assertion. - `test_submodule::test_root_module` does `assert len(rsmsp) >= 2` on a similar traversal result, failing. The race itself is non-deterministic, but the mapping from race outcome to exception type is deterministic per test. So each test has exactly two possible failure types, one per side of the race: - `test_docs`: `ValueError` -> Python wins, OR `IndexError` -> git wins. - `test_repo`: `ValueError` -> Python wins, OR `AssertionError` -> git wins. - `test_submodule`: `ValueError` -> Python wins, OR `AssertionError` -> git wins. In particular, `test_docs` never produces `AssertionError`, and `test_repo` and `test_submodule` never produce `IndexError`. Empirical confirmation ---------------------- In 1024 `reproduce-safe-dir` jobs and 4 buggy-config "test (fast)" job runs, 100% of 3084 target-test failures match the per-test prediction. `ValueError` accounts for ~99.0%, while the race-win exception types from the list above account for the others. `reproduce-safe-dir` runs: https://github.com/EliahKagan/GitPython/actions/runs/25836741324 https://github.com/EliahKagan/GitPython/actions/runs/25836329241 https://github.com/EliahKagan/GitPython/actions/runs/25836334196 https://github.com/EliahKagan/GitPython/actions/runs/25836339345 Run on the fix commit (256 jobs, 768 test outcomes, all PASSED): https://github.com/EliahKagan/GitPython/actions/runs/25836344886 The race condition, from a "test (fast)" job in a PR #2143 CI run: https://github.com/gitpython-developers/GitPython/actions/runs/25440735020/attempts/1 Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/cygwin-test.yml | 65 +++++++++++++++++++++++++------ test/test_docs.py | 8 ---- test/test_repo.py | 5 --- test/test_submodule.py | 5 --- 4 files changed, 53 insertions(+), 30 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 327e1f10c..e62106a18 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -20,46 +20,53 @@ jobs: runs-on: windows-latest - env: + env: &cygwin-env CHERE_INVOKING: "1" CYGWIN_NOWINPATH: "1" - defaults: + defaults: &cygwin-defaults run: shell: C:\cygwin\bin\bash.exe --login --norc -eo pipefail -o igncr "{0}" steps: - - name: Force LF line endings + - &force-lf + name: Force LF line endings run: | git config --global core.autocrlf false # Affects the non-Cygwin git. shell: pwsh # Do this outside Cygwin, to affect actions/checkout. - - uses: actions/checkout@v6 + - &checkout + uses: actions/checkout@v6 with: fetch-depth: 0 - - name: Install Cygwin + - &install-cygwin + name: Install Cygwin uses: cygwin/cygwin-install-action@v6 with: packages: git python39 python-pip-wheel python-setuptools-wheel python-wheel-wheel add-to-path: false # No need to change $PATH outside the Cygwin environment. - - name: Arrange for verbose output + - &verbose-output + name: Arrange for verbose output run: | # Arrange for verbose output but without shell environment setup details. echo 'set -x' >~/.bash_profile - - name: Special configuration for Cygwin git + - &safe-directory + name: Special configuration for Cygwin git run: | git config --global --add safe.directory "$(pwd)" git config --global --add safe.directory "$(pwd)/.git" git config --global core.autocrlf false - - name: Prepare this repo for tests + - &prepare-repo + name: Prepare this repo for tests run: | ./init-tests-after-clone.sh - - name: Set git user identity and command aliases for the tests + - &git-identity + name: Set git user identity and command aliases for the tests run: | git config --global user.email "travis@ci.com" git config --global user.name "Travis Runner" @@ -67,16 +74,19 @@ jobs: # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig - - name: Set up virtual environment + - &setup-venv + name: Set up virtual environment run: | python3.9 -m venv .venv echo 'BASH_ENV=.venv/bin/activate' >>"$GITHUB_ENV" - - name: Update PyPA packages + - &update-pypa + name: Update PyPA packages run: | python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel - - name: Install project and test dependencies + - &install-deps + name: Install project and test dependencies run: | pip install '.[test]' @@ -91,3 +101,34 @@ jobs: - name: Test with pytest (${{ matrix.additional-pytest-args }}) run: | pytest --color=yes -p no:sugar --instafail -vv ${{ matrix.additional-pytest-args }} + + reproduce-safe-dir: + strategy: + matrix: + run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256] + fail-fast: false + + runs-on: windows-latest + + env: *cygwin-env + + defaults: *cygwin-defaults + + steps: + - *force-lf + - *checkout + - *install-cygwin + - *verbose-output + - *safe-directory + - *prepare-repo + - *git-identity + - *setup-venv + - *update-pypa + - *install-deps + + - name: Run submodule tests + run: | + python -m pytest -vv \ + test/test_docs.py::Tutorials::test_submodules \ + test/test_repo.py::TestRepo::test_submodules \ + test/test_submodule.py::TestSubmodule::test_root_module diff --git a/test/test_docs.py b/test/test_docs.py index cc0bbf26a..c3426a807 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -6,9 +6,6 @@ import gc import os import os.path -import sys - -import pytest from test.lib import TestBase from test.lib.helper import with_rw_directory @@ -478,11 +475,6 @@ def test_references_and_objects(self, rw_dir): repo.git.clear_cache() - @pytest.mark.xfail( - sys.platform == "cygwin", - reason="Cygwin GitPython can't find SHA for submodule", - raises=ValueError, - ) def test_submodules(self): # [1-test_submodules] repo = self.rorepo diff --git a/test/test_repo.py b/test/test_repo.py index 13bff52e9..d2dd1ea5d 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -877,11 +877,6 @@ def test_repo_odbtype(self): target_type = GitCmdObjectDB self.assertIsInstance(self.rorepo.odb, target_type) - @pytest.mark.xfail( - sys.platform == "cygwin", - reason="Cygwin GitPython can't find submodule SHA", - raises=ValueError, - ) def test_submodules(self): self.assertEqual(len(self.rorepo.submodules), 1) # non-recursive self.assertGreaterEqual(len(list(self.rorepo.iter_submodules())), 2) diff --git a/test/test_submodule.py b/test/test_submodule.py index 63bb007de..0373e26f8 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -480,11 +480,6 @@ def test_base_rw(self, rwrepo): def test_base_bare(self, rwrepo): self._do_base_tests(rwrepo) - @pytest.mark.xfail( - sys.platform == "cygwin", - reason="Cygwin GitPython can't find submodule SHA", - raises=ValueError, - ) @pytest.mark.xfail( HIDE_WINDOWS_KNOWN_ERRORS, reason=( From 76f0160681f551b5199edd6dfd005012e65ad6d3 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 May 2026 14:33:14 -0400 Subject: [PATCH 1335/1392] Add a diagnostic job demonstrating the TokenOwner mechanism Temporarily add a `diag-token` job to the `cygwin-test` workflow that creates a directory through four code paths and reports its NTFS Owner. The idea is to empirically establish the cause of the Owner asymmetry: a Cygwin or Cygwin-derived runtime (`cygwin1.dll` for Cygwin git; `msys-2.0.dll` loaded by Git for Windows's bundled MSYS2 `sh.exe`) rewrites the process token's `TokenOwner` field at DLL initialization, and `CreateProcessW` propagates that mutation to every descendant. The four tests are: | Test | Sequence | Predicted Owner | | ---- | ----------------------------------------------- | ------------------------ | | A | PowerShell `New-Item` | `BUILTIN\Administrators` | | B | Cygwin `mkdir` | `runneradmin` (197108) | | C | Cygwin bash -> Git for Windows `git init` | `runneradmin` (197108) | | D | PowerShell -> Git Bash -> PowerShell `New-Item` | `runneradmin` (197108) | Test A is the Win32 baseline. The kernel-default `TokenOwner` for `runneradmin`'s full administrative token is `BUILTIN\Administrators`, because `runneradmin` is the built-in local Administrator (RID 500) and `FilterAdministratorToken=0` exempts it from UAC token filtering. Test B is the Cygwin baseline. Cygwin's `cygheap_user::init` calls `NtSetInformationToken(hProcToken, TokenOwner, &effec_cygsid, ...)` at DLL init, and the resulting Owner reflects the rewritten token. Test C is the load-bearing case. The child is a native Windows program that loads no Cygwin runtime of its own. It produces a user-owned `.git`, showing that the rewrite performed by the parent Cygwin bash propagates through `CreateProcessW`'s token duplication to a native Windows descendant. Test D strengthens the case: the first and last links are native Windows programs (a fresh PowerShell using its built-in `New-Item` cmdlet). Only the middle link is a Cygwin-family runtime (Git for Windows's MSYS2 `bash.exe`, linked against `msys-2.0.dll`). The directory is still user-owned, showing that the mechanism does not depend on git, nor on shell dispatch via `git submodule`, nor on any property of the final-link binary, but only on whether *any* process in the ancestry has loaded a Cygwin-family runtime. This `diag-token` job, like the `reproduce-safe-dir` matrix, is temporary and should be removed before this work is integrated. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/cygwin-test.yml | 67 +++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index e62106a18..9cc90804e 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -102,6 +102,73 @@ jobs: run: | pytest --color=yes -p no:sugar --instafail -vv ${{ matrix.additional-pytest-args }} + diag-token: + runs-on: windows-latest + + env: *cygwin-env + + defaults: *cygwin-defaults + + steps: + - *force-lf + - *checkout + - *install-cygwin + + - name: PowerShell-side token state and file-creation tests + shell: pwsh + run: | + $repo = "D:\a\GitPython\GitPython" + Write-Host "==================== whoami /all (PowerShell) ====================" + whoami /all + Write-Host "" + Write-Host "==================== Test A: PowerShell New-Item directory ====================" + $td = "$repo\test-pwsh-mkdir" + New-Item -ItemType Directory -Path $td -Force | Out-Null + $acl = Get-Acl -LiteralPath $td + Write-Host "Owner of $td : $($acl.Owner)" + Remove-Item $td -Force + Write-Host "" + Write-Host "==================== Test D: PowerShell -> Git Bash -> PowerShell New-Item ====================" + $td4 = "$repo\test-pwsh-bash-pwsh-mkdir" + $scriptPath = "$repo\inner-mkdir.ps1" + "New-Item -ItemType Directory -Path '$td4' -Force | Out-Null" | + Set-Content -Path $scriptPath -Encoding utf8 + $env:MSYS2_ARG_CONV_EXCL = '*' + try { + & "C:\Program Files\Git\bin\bash.exe" -c "powershell.exe -NoProfile -ExecutionPolicy Bypass -File '$scriptPath'" 2>&1 | Out-Null + } finally { + Remove-Item Env:MSYS2_ARG_CONV_EXCL -ErrorAction SilentlyContinue + } + if (Test-Path -LiteralPath $td4) { + $acl = Get-Acl -LiteralPath $td4 + Write-Host "Owner of $td4 (PowerShell -> bash -> PowerShell New-Item) : $($acl.Owner)" + Remove-Item $td4 -Force + } else { + Write-Host "Owner of $td4 (PowerShell -> bash -> PowerShell New-Item) : (directory not created)" + } + Remove-Item $scriptPath -Force -ErrorAction SilentlyContinue + + - name: Cygwin-side token state and file-creation tests + run: | + set +e + echo "==================== id (Cygwin) ====================" + id + echo + echo "==================== Test B: Cygwin mkdir ====================" + td="$(pwd)/test-cygwin-mkdir" + mkdir "$td" + echo "Owner: $(stat -c '%U(%u)' "$td")" + rmdir "$td" + echo + echo "==================== Test C: Cygwin-spawned Git for Windows git init ====================" + td3="$(pwd)/test-cygwin-spawns-wingit" + mkdir "$td3" + ( cd "$td3" && /cygdrive/c/Program\ Files/Git/bin/git.exe init -q ) + echo "Owner of $td3 (Cygwin-mkdir) : $(stat -c '%U(%u)' "$td3")" + echo "Owner of $td3/.git (Cygwin->Win git init): $(stat -c '%U(%u)' "$td3/.git")" + rm -rf "$td3" + true + reproduce-safe-dir: strategy: matrix: From f368f17d161854cf05d871ef8f48a22ab194bc63 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 May 2026 14:34:47 -0400 Subject: [PATCH 1336/1392] Show file ownership and safe.directory entries on every test job In each workflow that runs the test suite, add a step to the `test` job that prints the ownership (NTFS Owner / POSIX uid+gid) of key paths: the workspace, its `.git`, gitdb and smmap submodule worktrees and gitfiles, and the runner `HOME` and `~/.gitconfig`. Also print the full list of `safe.directory` entries at that point. GitPython's tests are intentionally coupled to the layout and state of the GitPython repository they run against. The ownership and trust config that gate whether git will operate on a path are part of that state. Surfacing them in every test job gives diagnostics to read alongside any failure that turns out to be a CI setup problem. The step is added to: - `cygwin-test.yml`'s `test` job, with a YAML anchor (`&ownership-display`) so the temporary `reproduce-safe-dir` matrix job can reuse it via `*ownership-display`. - `pythonpackage.yml`'s `test` job (Linux / macOS / native Windows). - `alpine-test.yml`'s `test` job. It runs after `init-tests-after-clone.sh` has populated the submodules and after `safe.directory` has been configured (in workflows that configure it), so the values reported are what the tests will see. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/alpine-test.yml | 20 ++++++++++ .github/workflows/cygwin-test.yml | 60 +++++++++++++++++++++++++++++ .github/workflows/pythonpackage.yml | 20 ++++++++++ 3 files changed, 100 insertions(+) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index b7de7482e..bb297efab 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -61,6 +61,26 @@ jobs: . .venv/bin/activate pip install '.[test]' + - name: Show file ownership and safe.directory entries + run: | + echo "==================== File ownership ====================" + for p in \ + "$(pwd)" \ + "$(pwd)/.git" \ + "$(pwd)/git/ext/gitdb" \ + "$(pwd)/git/ext/gitdb/.git" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ + "${HOME:-}" \ + "${HOME:-}/.gitconfig"; do + if [ -n "$p" ]; then + ls -ld -- "$p" 2>/dev/null || echo "(missing: $p)" + fi + done + echo + echo "==================== safe.directory entries ====================" + git config --global --get-all safe.directory 2>&1 || echo "(none)" + - name: Show version and platform information run: | . .venv/bin/activate diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 9cc90804e..6af9d098d 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -90,6 +90,65 @@ jobs: run: | pip install '.[test]' + - &ownership-display + name: Show file ownership and safe.directory entries + run: | + echo "==================== File ownership (Cygwin view, ls -ld) ====================" + for p in \ + "$(pwd)" \ + "$(pwd)/.git" \ + "$(pwd)/git/ext/gitdb" \ + "$(pwd)/git/ext/gitdb/.git" \ + "$(pwd)/.git/modules/gitdb" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ + "$(pwd)/.git/modules/gitdb/modules/smmap" \ + "${HOME:-}" \ + "${HOME:-}/.gitconfig"; do + if [ -n "$p" ]; then + ls -ld -- "$p" 2>/dev/null || echo "(missing: $p)" + fi + done + echo + echo "==================== File ownership (Win32 view, Get-Acl) ====================" + # Cygwin's ls -ld and stat go through Cygwin's SID-to-uid mapping + # (well-known SIDs by their RID, machine-local accounts by 0x30000+RID). + # The mapping is deterministic, but going via Win32 Get-Acl gives the + # NTAccount form of the NTFS Owner SID directly, with no Cygwin layer + # in between -- useful for confirming that what Cygwin reports as + # "Administrators" really is the BUILTIN\Administrators SID (S-1-5-32-544) + # rather than some local-machine account that Cygwin happens to map to + # the same uid. The workflow sets CYGWIN_NOWINPATH=1, so Windows paths + # are not on Cygwin's $PATH; invoke powershell.exe by absolute path. + ps_exe=/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0/powershell.exe + if [ -x "$ps_exe" ]; then + for p in \ + "$(pwd)" \ + "$(pwd)/.git" \ + "$(pwd)/git/ext/gitdb" \ + "$(pwd)/git/ext/gitdb/.git" \ + "$(pwd)/.git/modules/gitdb" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ + "$(pwd)/.git/modules/gitdb/modules/smmap" \ + "${HOME:-}/.gitconfig"; do + if [ -n "$p" ] && [ -e "$p" ]; then + wp=$(cygpath -w "$p") + # Escape single-quotes for PowerShell single-quoted string literal: ' -> '' + wp_escaped=${wp//\'/\'\'} + owner=$("$ps_exe" -NoProfile -NonInteractive -Command \ + "try { (Get-Acl -LiteralPath '${wp_escaped}').Owner } catch { 'ERROR: ' + \$_.Exception.Message }" \ + 2>/dev/null | tr -d '\r') + printf " %-44s %s\n" "$owner" "$wp" + fi + done + else + echo "($ps_exe not found -- skipping Win32 view)" + fi + echo + echo "==================== safe.directory entries ====================" + git config --global --get-all safe.directory 2>&1 || echo "(none)" + - name: Show version and platform information run: | uname -a @@ -192,6 +251,7 @@ jobs: - *setup-venv - *update-pypa - *install-deps + - *ownership-display - name: Run submodule tests run: | diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 874e18a8f..3d0dfede0 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -87,6 +87,26 @@ jobs: run: | pip install '.[test]' + - name: Show file ownership and safe.directory entries + run: | + echo "==================== File ownership ====================" + for p in \ + "$(pwd)" \ + "$(pwd)/.git" \ + "$(pwd)/git/ext/gitdb" \ + "$(pwd)/git/ext/gitdb/.git" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ + "${HOME:-}" \ + "${HOME:-}/.gitconfig"; do + if [ -n "$p" ]; then + ls -ld -- "$p" 2>/dev/null || echo "(missing: $p)" + fi + done + echo + echo "==================== safe.directory entries ====================" + git config --global --get-all safe.directory 2>&1 || echo "(none)" + - name: Show version and platform information run: | uname -a From 14cdc52fe97bbee17974ddb687b87b3c09af608c Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 May 2026 17:25:51 -0400 Subject: [PATCH 1337/1392] Restructure CI diagnostic steps Cleanups on the ownership-display step from the previous commit: - Split into per-shell steps (bash POSIX `ls -ld`, pwsh NTFS `Get-Acl`, bash safe.directory `git config`), removing the bash-driven PowerShell subprocess with `cygpath -w` and quote-escaping. Use `pwsh`, not `powershell.exe`. Gate pythonpackage's two views by `matrix.os-type` (Git Bash's `ls -ld` on Windows reports a uniform `runneradmin 197121` for every path, ignoring NTFS Owner -- MSYS2's SID-to-uid mapping doesn't have Cygwin's fidelity). - Trim the decorative `$HOME` entry from POSIX path lists: it isn't part of any git trust decision -- only `~/.gitconfig` is. Use `${HOME:?HOME is not set}/.gitconfig` for the remaining entry so an unset HOME fails loudly. - Move the pwsh path list into a `$paths = @(...)` variable. Unix shells keep the inline `for p in WORD WORD ...` form: alpine's `sh` (busybox ash) doesn't support arrays, and the others shouldn't differ from it unnecessarily. - Drop `2>&1` from the safe.directory step. Drop the `|| echo "(none)"` fallback on cygwin (entries are explicitly configured; bare command fails on regression). Keep it on pythonpackage and alpine, where `actions/checkout`'s safe.directory add lives under a throwaway HOME override and doesn't persist, so there legitimately are no entries to display. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/alpine-test.yml | 35 +++++---- .github/workflows/cygwin-test.yml | 112 ++++++++++++++-------------- .github/workflows/pythonpackage.yml | 70 +++++++++++++---- 3 files changed, 129 insertions(+), 88 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index bb297efab..4183f0e0d 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -61,25 +61,28 @@ jobs: . .venv/bin/activate pip install '.[test]' - - name: Show file ownership and safe.directory entries + - name: Show POSIX file ownership run: | - echo "==================== File ownership ====================" for p in \ - "$(pwd)" \ - "$(pwd)/.git" \ - "$(pwd)/git/ext/gitdb" \ - "$(pwd)/git/ext/gitdb/.git" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ - "${HOME:-}" \ - "${HOME:-}/.gitconfig"; do - if [ -n "$p" ]; then - ls -ld -- "$p" 2>/dev/null || echo "(missing: $p)" - fi + "$(pwd)" \ + "$(pwd)/.git" \ + "$(pwd)/git/ext/gitdb" \ + "$(pwd)/git/ext/gitdb/.git" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ + "${HOME:?HOME is not set}/.gitconfig" + do + ls -ld -- "$p" 2>/dev/null || echo "(missing: $p)" done - echo - echo "==================== safe.directory entries ====================" - git config --global --get-all safe.directory 2>&1 || echo "(none)" + + - name: Show safe.directory entries + # `actions/checkout`'s safe.directory add is only durable for the + # checkout itself (it writes under a throwaway HOME override and + # then discards it), so by the time this step runs the runner + # user's `~/.gitconfig` has no entries -- and the Alpine container + # chowns the workspace to runner:docker to match the test user, so + # git accepts the ownership without one. Expected: `(none)`. + run: git config --global --get-all safe.directory || echo "(none)" - name: Show version and platform information run: | diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 6af9d098d..16b88851e 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -90,64 +90,62 @@ jobs: run: | pip install '.[test]' - - &ownership-display - name: Show file ownership and safe.directory entries + - &ownership-posix-display + name: Show POSIX file ownership + # Cygwin's `ls -ld` reports the NTFS Owner SID via Cygwin's SID-to-uid + # mapping (well-known SIDs by their RID, machine-local accounts by + # 0x30000+RID). That mapping is what Cygwin git's + # `is_path_owned_by_current_user` reduces to, so this is the view that + # determines whether `safe.directory` is consulted. run: | - echo "==================== File ownership (Cygwin view, ls -ld) ====================" for p in \ - "$(pwd)" \ - "$(pwd)/.git" \ - "$(pwd)/git/ext/gitdb" \ - "$(pwd)/git/ext/gitdb/.git" \ - "$(pwd)/.git/modules/gitdb" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ - "$(pwd)/.git/modules/gitdb/modules/smmap" \ - "${HOME:-}" \ - "${HOME:-}/.gitconfig"; do - if [ -n "$p" ]; then - ls -ld -- "$p" 2>/dev/null || echo "(missing: $p)" - fi + "$(pwd)" \ + "$(pwd)/.git" \ + "$(pwd)/git/ext/gitdb" \ + "$(pwd)/git/ext/gitdb/.git" \ + "$(pwd)/.git/modules/gitdb" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ + "$(pwd)/.git/modules/gitdb/modules/smmap" \ + "${HOME:?HOME is not set}/.gitconfig" + do + ls -ld -- "$p" 2>/dev/null || echo "(missing: $p)" done - echo - echo "==================== File ownership (Win32 view, Get-Acl) ====================" - # Cygwin's ls -ld and stat go through Cygwin's SID-to-uid mapping - # (well-known SIDs by their RID, machine-local accounts by 0x30000+RID). - # The mapping is deterministic, but going via Win32 Get-Acl gives the - # NTAccount form of the NTFS Owner SID directly, with no Cygwin layer - # in between -- useful for confirming that what Cygwin reports as - # "Administrators" really is the BUILTIN\Administrators SID (S-1-5-32-544) - # rather than some local-machine account that Cygwin happens to map to - # the same uid. The workflow sets CYGWIN_NOWINPATH=1, so Windows paths - # are not on Cygwin's $PATH; invoke powershell.exe by absolute path. - ps_exe=/cygdrive/c/Windows/System32/WindowsPowerShell/v1.0/powershell.exe - if [ -x "$ps_exe" ]; then - for p in \ - "$(pwd)" \ - "$(pwd)/.git" \ - "$(pwd)/git/ext/gitdb" \ - "$(pwd)/git/ext/gitdb/.git" \ - "$(pwd)/.git/modules/gitdb" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ - "$(pwd)/.git/modules/gitdb/modules/smmap" \ - "${HOME:-}/.gitconfig"; do - if [ -n "$p" ] && [ -e "$p" ]; then - wp=$(cygpath -w "$p") - # Escape single-quotes for PowerShell single-quoted string literal: ' -> '' - wp_escaped=${wp//\'/\'\'} - owner=$("$ps_exe" -NoProfile -NonInteractive -Command \ - "try { (Get-Acl -LiteralPath '${wp_escaped}').Owner } catch { 'ERROR: ' + \$_.Exception.Message }" \ - 2>/dev/null | tr -d '\r') - printf " %-44s %s\n" "$owner" "$wp" - fi - done - else - echo "($ps_exe not found -- skipping Win32 view)" - fi - echo - echo "==================== safe.directory entries ====================" - git config --global --get-all safe.directory 2>&1 || echo "(none)" + + - &ownership-ntfs-display + name: Show NTFS file ownership + # Authoritative NTFS Owner via Get-Acl, with no Cygwin SID-to-uid layer + # in between -- useful for confirming what the Cygwin view reports as + # "Administrators" is the BUILTIN\Administrators SID (S-1-5-32-544). + shell: pwsh + run: | + $paths = @( + "$pwd", + "$pwd\.git", + "$pwd\git\ext\gitdb", + "$pwd\git\ext\gitdb\.git", + "$pwd\.git\modules\gitdb", + "$pwd\git\ext\gitdb\gitdb\ext\smmap", + "$pwd\git\ext\gitdb\gitdb\ext\smmap\.git", + "$pwd\.git\modules\gitdb\modules\smmap", + "$env:USERPROFILE\.gitconfig" + ) + foreach ($p in $paths) { + if (Test-Path -LiteralPath $p) { + try { + $owner = (Get-Acl -LiteralPath $p).Owner + } catch { + $owner = "ERROR: $($_.Exception.Message)" + } + "{0,-44} {1}" -f $owner, $p + } else { + "(missing: $p)" + } + } + + - &safe-directory-display + name: Show safe.directory entries + run: git config --global --get-all safe.directory - name: Show version and platform information run: | @@ -251,7 +249,9 @@ jobs: - *setup-venv - *update-pypa - *install-deps - - *ownership-display + - *ownership-posix-display + - *ownership-ntfs-display + - *safe-directory-display - name: Run submodule tests run: | diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3d0dfede0..cffafc59a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -87,25 +87,63 @@ jobs: run: | pip install '.[test]' - - name: Show file ownership and safe.directory entries + - name: Show POSIX file ownership + # Linux and macOS only. On Windows, Git Bash's `ls -ld` reports a + # uniform uid+gid for every path regardless of NTFS Owner (MSYS2's + # SID-to-uid mapping doesn't have Cygwin's fidelity), so it would + # not be informative here. The NTFS Owner check below covers Windows. + if: matrix.os-type != 'windows' run: | - echo "==================== File ownership ====================" for p in \ - "$(pwd)" \ - "$(pwd)/.git" \ - "$(pwd)/git/ext/gitdb" \ - "$(pwd)/git/ext/gitdb/.git" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ - "${HOME:-}" \ - "${HOME:-}/.gitconfig"; do - if [ -n "$p" ]; then - ls -ld -- "$p" 2>/dev/null || echo "(missing: $p)" - fi + "$(pwd)" \ + "$(pwd)/.git" \ + "$(pwd)/git/ext/gitdb" \ + "$(pwd)/git/ext/gitdb/.git" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ + "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ + "${HOME:?HOME is not set}/.gitconfig" + do + ls -ld -- "$p" 2>/dev/null || echo "(missing: $p)" done - echo - echo "==================== safe.directory entries ====================" - git config --global --get-all safe.directory 2>&1 || echo "(none)" + + - name: Show NTFS file ownership + # Windows only. Reads NTFS Owner directly via Get-Acl, which is the + # authoritative view for Windows-side ownership questions; the POSIX + # view via Git Bash's MSYS2 layer is not a reliable proxy here. + if: matrix.os-type == 'windows' + shell: pwsh + run: | + $paths = @( + "$pwd", + "$pwd\.git", + "$pwd\git\ext\gitdb", + "$pwd\git\ext\gitdb\.git", + "$pwd\git\ext\gitdb\gitdb\ext\smmap", + "$pwd\git\ext\gitdb\gitdb\ext\smmap\.git", + "$env:USERPROFILE\.gitconfig" + ) + foreach ($p in $paths) { + if (Test-Path -LiteralPath $p) { + try { + $owner = (Get-Acl -LiteralPath $p).Owner + } catch { + $owner = "ERROR: $($_.Exception.Message)" + } + "{0,-44} {1}" -f $owner, $p + } else { + "(missing: $p)" + } + } + + - name: Show safe.directory entries + # `actions/checkout`'s safe.directory add is only durable for the + # checkout itself (it writes under a throwaway HOME override and + # then discards it), so by the time this step runs the runner + # user's `~/.gitconfig` has no entries -- and git accepts the + # workspace's ownership anyway: Git for Windows via its + # Admins-group exemption on the windows matrix; on Linux/macOS + # the workspace is owned by the test user. Expected: `(none)`. + run: git config --global --get-all safe.directory || echo "(none)" - name: Show version and platform information run: | From 8ef4c21859d61683ac93f679790c23ef1cf4b17d Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 May 2026 14:35:14 -0400 Subject: [PATCH 1338/1392] Add `submodules: recursive` temporarily, for testing Add `submodules: recursive` to the `actions/checkout` step in every workflow that runs the test suite (`cygwin-test.yml`, `pythonpackage.yml`, `alpine-test.yml`). This is *temporary* and will be reverted after the bugfix, with the explicit intent of demonstrating that the bugfix works regardless of which mechanism populates the submodules. The standing decision is to NOT use `submodules: recursive` in CI. `init-tests-after-clone.sh` is the documented setup mechanism that downstream packagers (Arch Linux and others) rely on, and keeping it as the sole submodule source on upstream CI is meant to catch regressions like #1713 before they reach distros. See #1715 for the full rationale. The CI run on this commit is expected to show: - Cygwin (`test-cygwin`): the `safe.directory` bug still triggers, with the same `ValueError`/`IndexError`/`AssertionError` pattern as the previous commits. The bug is independent of which process clones the submodules; the gitdb worktree directory itself is created Admin-owned by the outer `git clone`'s checkout phase before any submodule init runs, regardless of which mechanism populates the submodule contents afterward. - Linux / macOS / native Windows (`Python package`) and Alpine Linux (`test-alpine`): tests pass as before. These platforms are unaffected by the bug. Each workflow's checkout step carries an inline comment pointing at #1715 so the temporary nature of the change is legible at a glance. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/alpine-test.yml | 3 +++ .github/workflows/cygwin-test.yml | 3 +++ .github/workflows/pythonpackage.yml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 4183f0e0d..259a6d9f2 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -29,6 +29,9 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 + # Temporary, for testing. The standing decision is to NOT pre-clone + # submodules on CI; see https://github.com/gitpython-developers/GitPython/pull/1715 + submodules: recursive - name: Set workspace ownership run: | diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 16b88851e..f5e1309e9 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -39,6 +39,9 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 + # Temporary, for testing. The standing decision is to NOT pre-clone + # submodules on CI; see https://github.com/gitpython-developers/GitPython/pull/1715 + submodules: recursive - &install-cygwin name: Install Cygwin diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index cffafc59a..919a32721 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -53,6 +53,9 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 + # Temporary, for testing. The standing decision is to NOT pre-clone + # submodules on CI; see https://github.com/gitpython-developers/GitPython/pull/1715 + submodules: recursive - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 From b409946ca7e7c36782b47b54dd5b1f70280e8eab Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Wed, 6 May 2026 23:06:57 -0400 Subject: [PATCH 1339/1392] Cover submodule working trees in safe.directory on Cygwin Add the gitdb and smmap submodule working tree paths to the `safe.directory` config in the Cygwin CI workflow. Without this, when GitPython opens a submodule as a `Repo` and runs `git cat-file --batch-check` against it, git rejects the repository for dubious ownership. The user-visible failure modes (`ValueError`, `IndexError`, `AssertionError`) all trace back to this rejection. Why `[gitdb]` failed and `[smmap]` passed ----------------------------------------- The trust check in `test_fixture_health.py` failed for `[gitdb]` but passed for `[smmap]`, even though neither submodule's working tree was in the workflow's `safe.directory` list before this commit. The asymmetry comes down to which Owner SID NTFS records on each path, and which paths git's ownership check requires to be owned. There are six paths git's check might consider for the two submodules: gitdb's `.git` gitfile, worktree `git/ext/gitdb`, and gitdir `/.git/modules/gitdb`; smmap's `.git` gitfile, worktree `git/ext/gitdb/gitdb/ext/smmap`, and gitdir `/.git/modules/gitdb/modules/smmap`. Of these, only one had NTFS Owner = `BUILTIN\Administrators` (Cygwin uid 544): gitdb's worktree. The other five had Owner = `runneradmin`, whose Cygwin uid (197108) was the value `geteuid()` returned in the test process. The same Owner pattern held both with and without `submodules: recursive` in `actions/checkout`. The single `Administrators`-owned path was gitdb's worktree. All other paths were `runneradmin`-owned, including the ones that Git for Windows's recursive submodule clone had just produced when `submodules: recursive` was set. The differentiator is not which git binary clones the submodules, but that `git/ext/gitdb` is created by the outer `git clone`'s checkout phase. When `git checkout` materializes a tree entry of mode 160000, it calls `mkdir(path, 0777)` to create an empty submodule directory (see `entry.c::write_entry`, case `S_IFGITLINK` [1] [2]). On Windows GHA runners, jobs run as `runneradmin`. This is the built-in local Administrator account (its Cygwin uid 197108 = 196608 + 500 matches Cygwin's mapping [3] for machine-local accounts: 0x30000 plus the SID suffix 500, the well-known suffix of that account's SID). That account is exempt from UAC token filtering by default (Admin Approval Mode for the built-in Administrator account is disabled [4]), so its processes hold the full administrative access token. `CreateProcessW` propagates the parent's primary token unchanged through `actions/checkout`'s process tree. Inside that tree, the outer `git clone`'s `mkdir(path, 0777)` produces directories whose NTFS Owner is `BUILTIN\Administrators` -- as observed on every workspace directory the outer clone materialized, including the `git/ext/gitdb` placeholder. Subsequent submodule-update operations -- both Git for Windows if `actions/checkout` does a recursive clone, and Cygwin git if it happens later due to `init-tests-after-clone.sh` -- produce paths that NTFS records as `runneradmin`-owned. Both flows go through a process whose primary token's `TokenOwner` field has been rewritten from `BUILTIN\Administrators` to the user SID by a Cygwin or Cygwin-derived runtime at DLL initialization. The rewrite propagates to every descendant via `CreateProcessW`'s primary-token inheritance [5], so every `mkdir` issued after that point produces a directory owned by the user. - Cygwin git triggers the rewrite directly. `cygheap_user::init` in `cygwin1.dll` calls `NtSetInformationToken(hProcToken, TokenOwner, &effec_cygsid, ...)` at process startup [6]. - Git for Windows triggers it indirectly. `git submodule` is not a builtin (only `submodule--helper` is) [7]. So it falls through to `execv_dashed_external` and runs `git-submodule.sh`, a shell script whose shebang is resolved at runtime to `sh.exe` in the Git for Windows "Git Bash" environment. That `sh.exe` is an MSYS2 binary linked against `msys-2.0.dll`, a Cygwin fork that performs the same `TokenOwner` rewrite. From there, every `git.exe` the script spawns inherits the user-SID `TokenOwner` and produces user-owned directories. Cygwin's `lstat().st_uid` reports the actual NTFS Owner SID mapped through Cygwin's SID-to-uid table. `is_path_owned_by_current_user` reduces to `lstat(p).st_uid == geteuid()` on Cygwin (no Administrators group exemption). `ensure_valid_ownership` returns 1 (accepting the repository) without consulting `safe.directory` when the gitfile, worktree, and gitdir ALL pass that owned-by-current-user check. Otherwise it falls through to comparing the worktree's `real_pathdup` against each configured `safe.directory` entry. For gitdb the three Owners were `runneradmin` (gitfile), **`Administrators` (worktree)**, and `runneradmin` (gitdir), so the all-paths-owned check failed on the worktree. The workflow's `safe.directory` before this commit contained only `$(pwd)` and `$(pwd)/.git`, neither of which exact-matches `git/ext/gitdb`, so the `safe.directory` comparison also failed, and `ensure_valid_ownership` returned 0 -- git rejected the repository. For smmap the three Owners were all `runneradmin`, so the all-paths-owned check passed. Cygwin's `chown` cannot rewrite the gitdb worktree's Owner SID from `Administrators` to `runneradmin`: it returns "Permission denied". Adding both submodule worktree paths to `safe.directory` is the correct fix and is robust against shifts in what paths inherit which Owner. Why `actions/checkout`'s own `safe.directory` does not help ----------------------------------------------------------- `actions/checkout`'s `set-safe-directory: true` default writes the main repository path to `safe.directory` in a temporary config it points its own spawned git child at by overriding `HOME` for that child process. That `HOME` override applies only to git invocations the action itself spawns; subsequent steps' processes inherit the runner user's real `HOME` (e.g., `C:\Users\runneradmin` on the Windows runner) and read its actual `~/.gitconfig`, which never received the entry. So no git in a later step, whether Git for Windows or Cygwin git, sees it. That's why the `cygwin-test` workflow sets Cygwin git's `safe.directory` itself. This commit extends that to cover the gitdb and smmap working trees. The distinction between Cygwin git and Git for Windows is also why the bug affected the Cygwin jobs and no other Windows jobs. `compat/mingw.c` defines `is_path_owned_by_current_sid` [8], which accepts `BUILTIN\Administrators`-owned paths when the current user is a member of `Administrators`. Cygwin git compiles against the POSIX path (`is_path_owned_by_current_uid` in `git-compat-util.h` [9]) without that leniency. So the same `Administrators`-owned `git/ext/gitdb` that Cygwin git rejects is silently accepted by Git for Windows, and the main CI workflow's `windows-latest` jobs never trip the trust check. Verification ------------ The `reproduce-safe-dir` matrix on the previous commits produces failures for all three affected tests; this commit's CI run shows those tests passing instead. The Owner-SID claim above is verified by the `diag-token` job introduced for this purpose. That job creates a directory through four code paths (PowerShell-only, Cygwin-only, Cygwin-bash spawning Win32 `git init`, and a PowerShell -> Cygwin-bash -> PowerShell sandwich) and reports the NTFS Owner of each. The observed Owners match the predicted values in every case, including the load-bearing Cygwin -> Win32 propagation case (test C) and the sandwich case (test D) showing that the determinant is whether some process in the ancestry has loaded a Cygwin-family runtime, not the identity of the file-creating binary. The commit immediately preceding this one temporarily sets `submodules: recursive` on `actions/checkout` in every workflow that runs the test suite. Its CI run shows the bug still triggering on Cygwin (the gitdb worktree directory itself is created by the outer `git clone`'s checkout phase, before any submodule init runs, regardless of which mechanism subsequently populates the submodule contents). A subsequent commit will revert that change; its CI run shows this fix continues to hold without `submodules: recursive`, confirming the fix is independent of submodule source. [1]: https://github.com/git/git/blob/v2.51.0/entry.c#L397 [2]: https://github.com/git-for-windows/git/blob/v2.54.0.windows.1/entry.c#L397 [3]: https://cygwin.com/cygwin-ug-net/ntsec.html [4]: https://learn.microsoft.com/en-us/windows/security/application-security/application-control/user-account-control/settings-and-configuration [5]: https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessw [6]: https://sourceware.org/cgit/newlib-cygwin/tree/winsup/cygwin/uinfo.cc?h=cygwin-3.6.9#n82 [7]: https://github.com/git-for-windows/git/blob/v2.54.0.windows.1/git.c#L661 [8]: https://github.com/git-for-windows/git/blob/v2.54.0.windows.1/compat/mingw.c#L3931 [9]: https://github.com/git/git/blob/v2.51.0/git-compat-util.h#L346 Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/cygwin-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index f5e1309e9..32b353d2e 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -61,6 +61,8 @@ jobs: run: | git config --global --add safe.directory "$(pwd)" git config --global --add safe.directory "$(pwd)/.git" + git config --global --add safe.directory "$(pwd)/git/ext/gitdb" + git config --global --add safe.directory "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" git config --global core.autocrlf false - &prepare-repo From 650eaafebd44cbe879d60eb8389eacc5db323c63 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 May 2026 14:35:57 -0400 Subject: [PATCH 1340/1392] Remove `submodules: recursive` (testing complete) Revert the temporary addition of `submodules: recursive` to `actions/checkout` in `cygwin-test.yml`, `pythonpackage.yml`, and `alpine-test.yml`. The `safe.directory` fix has been verified to work regardless of which mechanism populates the submodules. Returning the workflows to their pre-test posture restores the standing arrangement: `init-tests-after-clone.sh` as the sole submodule source on upstream CI. See #1715 for the rationale. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/alpine-test.yml | 3 --- .github/workflows/cygwin-test.yml | 3 --- .github/workflows/pythonpackage.yml | 3 --- 3 files changed, 9 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 259a6d9f2..4183f0e0d 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -29,9 +29,6 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 - # Temporary, for testing. The standing decision is to NOT pre-clone - # submodules on CI; see https://github.com/gitpython-developers/GitPython/pull/1715 - submodules: recursive - name: Set workspace ownership run: | diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 32b353d2e..caed9814b 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -39,9 +39,6 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 - # Temporary, for testing. The standing decision is to NOT pre-clone - # submodules on CI; see https://github.com/gitpython-developers/GitPython/pull/1715 - submodules: recursive - &install-cygwin name: Install Cygwin diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 919a32721..cffafc59a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -53,9 +53,6 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 - # Temporary, for testing. The standing decision is to NOT pre-clone - # submodules on CI; see https://github.com/gitpython-developers/GitPython/pull/1715 - submodules: recursive - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 From de3a950d57c8057fdd36ead97a390a48180892ee Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 10 May 2026 14:36:45 -0400 Subject: [PATCH 1341/1392] Remove temporary test jobs (testing complete) Remove three things from the `cygwin-test` workflow that were added to demonstrate the `safe.directory` bug and its fix: - The `reproduce-safe-dir` matrix (256 jobs running three submodule tests). Added to give a high-confidence reproduction of the failure pattern across runner-instance variation. - The `diag-token` job. Added to empirically establish the TokenOwner rewrite mechanism behind the gitdb-worktree Owner asymmetry. - The YAML anchors that only those temporary jobs needed (`&force-lf`, `&checkout`, `&install-cygwin`, `&verbose-output`, `&safe-directory`, `&prepare-repo`, `&git-identity`, `&setup-venv`, `&update-pypa`, `&install-deps`, `&ownership-posix-display`, `&ownership-ntfs-display`, `&safe-directory-display`, `&cygwin-env`, `&cygwin-defaults`). The `test` job still has all those steps; it just no longer needs to anchor them for reuse. What stays: the `test` job (the actual Cygwin test suite), the fixture-health and required-submodule checks, and the always-on file-ownership / `safe.directory` diagnostic steps (kept across all test workflows). Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/cygwin-test.yml | 144 ++++-------------------------- 1 file changed, 15 insertions(+), 129 deletions(-) diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index caed9814b..c12ccb3cf 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -20,41 +20,36 @@ jobs: runs-on: windows-latest - env: &cygwin-env + env: CHERE_INVOKING: "1" CYGWIN_NOWINPATH: "1" - defaults: &cygwin-defaults + defaults: run: shell: C:\cygwin\bin\bash.exe --login --norc -eo pipefail -o igncr "{0}" steps: - - &force-lf - name: Force LF line endings + - name: Force LF line endings run: | git config --global core.autocrlf false # Affects the non-Cygwin git. shell: pwsh # Do this outside Cygwin, to affect actions/checkout. - - &checkout - uses: actions/checkout@v6 + - uses: actions/checkout@v6 with: fetch-depth: 0 - - &install-cygwin - name: Install Cygwin + - name: Install Cygwin uses: cygwin/cygwin-install-action@v6 with: packages: git python39 python-pip-wheel python-setuptools-wheel python-wheel-wheel add-to-path: false # No need to change $PATH outside the Cygwin environment. - - &verbose-output - name: Arrange for verbose output + - name: Arrange for verbose output run: | # Arrange for verbose output but without shell environment setup details. echo 'set -x' >~/.bash_profile - - &safe-directory - name: Special configuration for Cygwin git + - name: Special configuration for Cygwin git run: | git config --global --add safe.directory "$(pwd)" git config --global --add safe.directory "$(pwd)/.git" @@ -62,13 +57,11 @@ jobs: git config --global --add safe.directory "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" git config --global core.autocrlf false - - &prepare-repo - name: Prepare this repo for tests + - name: Prepare this repo for tests run: | ./init-tests-after-clone.sh - - &git-identity - name: Set git user identity and command aliases for the tests + - name: Set git user identity and command aliases for the tests run: | git config --global user.email "travis@ci.com" git config --global user.name "Travis Runner" @@ -76,24 +69,20 @@ jobs: # and cause subsequent tests to fail cat test/fixtures/.gitconfig >> ~/.gitconfig - - &setup-venv - name: Set up virtual environment + - name: Set up virtual environment run: | python3.9 -m venv .venv echo 'BASH_ENV=.venv/bin/activate' >>"$GITHUB_ENV" - - &update-pypa - name: Update PyPA packages + - name: Update PyPA packages run: | python -m pip install -U pip 'setuptools; python_version<"3.12"' wheel - - &install-deps - name: Install project and test dependencies + - name: Install project and test dependencies run: | pip install '.[test]' - - &ownership-posix-display - name: Show POSIX file ownership + - name: Show POSIX file ownership # Cygwin's `ls -ld` reports the NTFS Owner SID via Cygwin's SID-to-uid # mapping (well-known SIDs by their RID, machine-local accounts by # 0x30000+RID). That mapping is what Cygwin git's @@ -114,8 +103,7 @@ jobs: ls -ld -- "$p" 2>/dev/null || echo "(missing: $p)" done - - &ownership-ntfs-display - name: Show NTFS file ownership + - name: Show NTFS file ownership # Authoritative NTFS Owner via Get-Acl, with no Cygwin SID-to-uid layer # in between -- useful for confirming what the Cygwin view reports as # "Administrators" is the BUILTIN\Administrators SID (S-1-5-32-544). @@ -145,8 +133,7 @@ jobs: } } - - &safe-directory-display - name: Show safe.directory entries + - name: Show safe.directory entries run: git config --global --get-all safe.directory - name: Show version and platform information @@ -160,104 +147,3 @@ jobs: - name: Test with pytest (${{ matrix.additional-pytest-args }}) run: | pytest --color=yes -p no:sugar --instafail -vv ${{ matrix.additional-pytest-args }} - - diag-token: - runs-on: windows-latest - - env: *cygwin-env - - defaults: *cygwin-defaults - - steps: - - *force-lf - - *checkout - - *install-cygwin - - - name: PowerShell-side token state and file-creation tests - shell: pwsh - run: | - $repo = "D:\a\GitPython\GitPython" - Write-Host "==================== whoami /all (PowerShell) ====================" - whoami /all - Write-Host "" - Write-Host "==================== Test A: PowerShell New-Item directory ====================" - $td = "$repo\test-pwsh-mkdir" - New-Item -ItemType Directory -Path $td -Force | Out-Null - $acl = Get-Acl -LiteralPath $td - Write-Host "Owner of $td : $($acl.Owner)" - Remove-Item $td -Force - Write-Host "" - Write-Host "==================== Test D: PowerShell -> Git Bash -> PowerShell New-Item ====================" - $td4 = "$repo\test-pwsh-bash-pwsh-mkdir" - $scriptPath = "$repo\inner-mkdir.ps1" - "New-Item -ItemType Directory -Path '$td4' -Force | Out-Null" | - Set-Content -Path $scriptPath -Encoding utf8 - $env:MSYS2_ARG_CONV_EXCL = '*' - try { - & "C:\Program Files\Git\bin\bash.exe" -c "powershell.exe -NoProfile -ExecutionPolicy Bypass -File '$scriptPath'" 2>&1 | Out-Null - } finally { - Remove-Item Env:MSYS2_ARG_CONV_EXCL -ErrorAction SilentlyContinue - } - if (Test-Path -LiteralPath $td4) { - $acl = Get-Acl -LiteralPath $td4 - Write-Host "Owner of $td4 (PowerShell -> bash -> PowerShell New-Item) : $($acl.Owner)" - Remove-Item $td4 -Force - } else { - Write-Host "Owner of $td4 (PowerShell -> bash -> PowerShell New-Item) : (directory not created)" - } - Remove-Item $scriptPath -Force -ErrorAction SilentlyContinue - - - name: Cygwin-side token state and file-creation tests - run: | - set +e - echo "==================== id (Cygwin) ====================" - id - echo - echo "==================== Test B: Cygwin mkdir ====================" - td="$(pwd)/test-cygwin-mkdir" - mkdir "$td" - echo "Owner: $(stat -c '%U(%u)' "$td")" - rmdir "$td" - echo - echo "==================== Test C: Cygwin-spawned Git for Windows git init ====================" - td3="$(pwd)/test-cygwin-spawns-wingit" - mkdir "$td3" - ( cd "$td3" && /cygdrive/c/Program\ Files/Git/bin/git.exe init -q ) - echo "Owner of $td3 (Cygwin-mkdir) : $(stat -c '%U(%u)' "$td3")" - echo "Owner of $td3/.git (Cygwin->Win git init): $(stat -c '%U(%u)' "$td3/.git")" - rm -rf "$td3" - true - - reproduce-safe-dir: - strategy: - matrix: - run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256] - fail-fast: false - - runs-on: windows-latest - - env: *cygwin-env - - defaults: *cygwin-defaults - - steps: - - *force-lf - - *checkout - - *install-cygwin - - *verbose-output - - *safe-directory - - *prepare-repo - - *git-identity - - *setup-venv - - *update-pypa - - *install-deps - - *ownership-posix-display - - *ownership-ntfs-display - - *safe-directory-display - - - name: Run submodule tests - run: | - python -m pytest -vv \ - test/test_docs.py::Tutorials::test_submodules \ - test/test_repo.py::TestRepo::test_submodules \ - test/test_submodule.py::TestSubmodule::test_root_module From 4dd89aff2896f63f8d7e61ef3736b1b60e386d16 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sun, 17 May 2026 15:49:25 -0400 Subject: [PATCH 1342/1392] Match test_root_module's deep-traversal assertion to gitdb's structure gitdb's `async` submodule was removed back in 2014 (gitpython-developers/gitdb@bf942a9); only smmap remains. The leading "gitdb / async" comment and the `assert len(rsmsp) >= 2` check (loosened back in 2011 from `== 2` in 4a8bdce7 when smmap was added to gitdb alongside async) are both stale. Replace with an exact list-equality check on the expected paths in traversal order. That order is also what later code in this function assumes via positional indexing `rsmsp[0]`, `rsmsp[1]`. Co-authored-by: Claude Opus 4.7 (1M context) --- test/test_submodule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_submodule.py b/test/test_submodule.py index 0373e26f8..778d22e3f 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -508,9 +508,9 @@ def test_root_module(self, rwrepo): with rm.config_writer(): pass - # Deep traversal gitdb / async. + # Deep traversal yields gitdb and its nested smmap. rsmsp = [sm.path for sm in rm.traverse()] - assert len(rsmsp) >= 2 # gitdb and async [and smmap], async being a child of gitdb. + assert rsmsp == ["git/ext/gitdb", "gitdb/ext/smmap"] # Cannot set the parent commit as root module's path didn't exist. self.assertRaises(ValueError, rm.set_parent_commit, "HEAD") From 38956826cbf8a4ce8c345285a9d8fc78ef17aa21 Mon Sep 17 00:00:00 2001 From: Deepak kudi Date: Thu, 21 May 2026 13:09:53 +0530 Subject: [PATCH 1343/1392] Support index diffs against empty tree Assisted-by: OpenAI GPT-5 --- git/index/base.py | 43 ++++++++++++++++++++++++++++++++++++++++--- test/test_index.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 2276343f2..93d4cda52 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1480,12 +1480,11 @@ def reset( return self - # FIXME: This is documented to accept the same parameters as Diffable.diff, but this - # does not handle NULL_TREE for `other`. (The suppressed mypy error is about this.) def diff( self, - other: Union[ # type: ignore[override] + other: Union[ Literal[git_diff.DiffConstants.INDEX], + Literal[git_diff.DiffConstants.NULL_TREE], "Tree", "Commit", str, @@ -1512,6 +1511,44 @@ def diff( if other is self.INDEX: return git_diff.DiffIndex() + if other is git_diff.NULL_TREE: + args: List[Union[PathLike, str]] = [ + "--cached", + "4b825dc642cb6eb9a060e54bf8d69288fbee4904", + "--abbrev=40", + "--full-index", + ] + + if not any(x in kwargs for x in ("find_renames", "no_renames", "M")): + args.append("-M") + + if create_patch: + args.append("-p") + args.append("--no-ext-diff") + else: + args.append("--raw") + args.append("-z") + + args.append("--no-color") + + if paths is not None and not isinstance(paths, (tuple, list)): + paths = [paths] + + if paths: + args.append("--") + args.extend(paths) + + kwargs["as_process"] = True + proc = self.repo.git.diff(*args, **kwargs) + + diff_method = ( + git_diff.Diff._index_from_patch_format if create_patch else git_diff.Diff._index_from_raw_format + ) + index = diff_method(self.repo, proc) + + proc.wait() + return index + # Index against anything but None is a reverse diff with the respective item. # Handle existing -R flags properly. # Transform strings to the object so that we can call diff on it. diff --git a/test/test_index.py b/test/test_index.py index cb45d3e90..327860d72 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -23,7 +23,7 @@ import ddt import pytest -from git import BlobFilter, Diff, Git, IndexFile, Object, Repo, Tree +from git import BlobFilter, Diff, Git, IndexFile, NULL_TREE, Object, Repo, Tree from git.exc import ( CheckoutError, GitCommandError, @@ -555,6 +555,34 @@ def test_index_file_diffing(self, rw_repo): rval = index.checkout("lib") assert len(list(rval)) > 1 + @with_rw_directory + def test_index_file_diff_null_tree_with_initial_index(self, rw_dir): + repo = Repo.init(rw_dir) + filename = ".gitkeep" + file_path = osp.join(repo.working_tree_dir, filename) + with open(file_path, "w") as fp: + fp.write("# Initial file\n") + + index = repo.index + index.add([filename]) + index.write() + + index = IndexFile(repo) + assert not index.diff(None) + + diff = index.diff(NULL_TREE) + self.assertEqual(len(diff), 1) + self.assertEqual(diff[0].change_type, "A") + assert diff[0].new_file + self.assertEqual(diff[0].b_path, filename) + + self.assertEqual(len(index.diff(NULL_TREE, paths=filename)), 1) + self.assertEqual(len(index.diff(NULL_TREE, paths="missing")), 0) + + patch = index.diff(NULL_TREE, create_patch=True) + self.assertEqual(len(patch), 1) + self.assertIn(b"+# Initial file", patch[0].diff) + def _count_existing(self, repo, files): """Return count of files that actually exist in the repository directory.""" existing = 0 From fe4b66dc621cf6d79da8cb95d20f775c8a373a93 Mon Sep 17 00:00:00 2001 From: Puneet Dixit <236133619+puneetdixit200@users.noreply.github.com> Date: Sat, 23 May 2026 20:34:42 +0530 Subject: [PATCH 1344/1392] fix: use shared empty tree sha for index diffs Assisted-by: ChatGPT --- git/diff.py | 5 ++++- git/index/base.py | 4 ++-- test/test_index.py | 4 +++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/git/diff.py b/git/diff.py index 23cb5675e..b10f3f106 100644 --- a/git/diff.py +++ b/git/diff.py @@ -3,7 +3,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -__all__ = ["DiffConstants", "NULL_TREE", "INDEX", "Diffable", "DiffIndex", "Diff"] +__all__ = ["DiffConstants", "NULL_TREE", "NULL_TREE_SHA", "INDEX", "Diffable", "DiffIndex", "Diff"] import enum import re @@ -84,6 +84,9 @@ class DiffConstants(enum.Enum): :const:`git.NULL_TREE` and :const:`Diffable.NULL_TREE`. """ +NULL_TREE_SHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" +"""SHA of Git's canonical empty tree object.""" + INDEX: Literal[DiffConstants.INDEX] = DiffConstants.INDEX """Stand-in indicating you want to diff against the index. diff --git a/git/index/base.py b/git/index/base.py index 93d4cda52..f03b452dc 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -1511,10 +1511,10 @@ def diff( if other is self.INDEX: return git_diff.DiffIndex() - if other is git_diff.NULL_TREE: + if other == git_diff.NULL_TREE or other == git_diff.NULL_TREE_SHA: args: List[Union[PathLike, str]] = [ "--cached", - "4b825dc642cb6eb9a060e54bf8d69288fbee4904", + git_diff.NULL_TREE_SHA, "--abbrev=40", "--full-index", ] diff --git a/test/test_index.py b/test/test_index.py index 327860d72..4a32dd5dc 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -24,6 +24,7 @@ import pytest from git import BlobFilter, Diff, Git, IndexFile, NULL_TREE, Object, Repo, Tree +from git.diff import NULL_TREE_SHA from git.exc import ( CheckoutError, GitCommandError, @@ -568,7 +569,7 @@ def test_index_file_diff_null_tree_with_initial_index(self, rw_dir): index.write() index = IndexFile(repo) - assert not index.diff(None) + self.assertEqual(len(index.diff(None)), 0) diff = index.diff(NULL_TREE) self.assertEqual(len(diff), 1) @@ -577,6 +578,7 @@ def test_index_file_diff_null_tree_with_initial_index(self, rw_dir): self.assertEqual(diff[0].b_path, filename) self.assertEqual(len(index.diff(NULL_TREE, paths=filename)), 1) + self.assertEqual(len(index.diff(NULL_TREE_SHA, paths=filename)), 1) self.assertEqual(len(index.diff(NULL_TREE, paths="missing")), 0) patch = index.diff(NULL_TREE, create_patch=True) From da07a64985290b76f6df4495faea8a569ca29288 Mon Sep 17 00:00:00 2001 From: Puneet Dixit <236133619+puneetdixit200@users.noreply.github.com> Date: Sun, 24 May 2026 00:00:40 +0530 Subject: [PATCH 1345/1392] test: allow alternate bad fetch errors Co-authored-by: OpenAI Codex --- test/test_remote.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_remote.py b/test/test_remote.py index 1c627127a..2230c8df4 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -687,7 +687,12 @@ def test_multiple_urls(self, rw_repo): def test_fetch_error(self): rem = self.rorepo.remote("origin") - with self.assertRaisesRegex(GitCommandError, "[Cc]ouldn't find remote ref __BAD_REF__"): + msg = ( + r"[Cc]ouldn't find remote ref __BAD_REF__|" + r"could not read Username|" + r"expected flush after ref listing" + ) + with self.assertRaisesRegex(GitCommandError, msg): rem.fetch("__BAD_REF__") @with_rw_repo("0.1.6", bare=False) From 59cc3bb1f5d43900f94f1c5044766e76b89f6445 Mon Sep 17 00:00:00 2001 From: Puneet Dixit <236133619+puneetdixit200@users.noreply.github.com> Date: Sun, 24 May 2026 00:13:13 +0530 Subject: [PATCH 1346/1392] fix: preserve diff process stderr Co-authored-by: OpenAI Codex --- git/diff.py | 18 +++++++++++++++--- test/test_index.py | 4 ++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/git/diff.py b/git/diff.py index b10f3f106..5af53e556 100644 --- a/git/diff.py +++ b/git/diff.py @@ -602,7 +602,14 @@ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoIn # FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise. text_list: List[bytes] = [] - handle_process_output(proc, text_list.append, None, finalize_process, decode_streams=False) + stderr_list: List[bytes] = [] + + def finalize_process_with_stderr(proc: Union["Popen", "Git.AutoInterrupt"]) -> None: + finalize_process(proc, stderr=b"".join(stderr_list)) + + handle_process_output( + proc, text_list.append, stderr_list.append, finalize_process_with_stderr, decode_streams=False + ) # For now, we have to bake the stream. text = b"".join(text_list) @@ -768,11 +775,16 @@ def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex[Diff] # :100644 100644 687099101... 37c5e30c8... M .gitignore index: "DiffIndex" = DiffIndex() + stderr_list: List[bytes] = [] + + def finalize_process_with_stderr(proc: Union["Popen", "Git.AutoInterrupt"]) -> None: + finalize_process(proc, stderr=b"".join(stderr_list)) + handle_process_output( proc, lambda byt: cls._handle_diff_line(byt, repo, index), - None, - finalize_process, + stderr_list.append, + finalize_process_with_stderr, decode_streams=False, ) diff --git a/test/test_index.py b/test/test_index.py index 4a32dd5dc..3be750dbb 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -585,6 +585,10 @@ def test_index_file_diff_null_tree_with_initial_index(self, rw_dir): self.assertEqual(len(patch), 1) self.assertIn(b"+# Initial file", patch[0].diff) + with self.assertRaises(GitCommandError) as exc_info: + index.diff(NULL_TREE, bogus_option=True) + self.assertIn("usage: git diff", exc_info.exception.stderr) + def _count_existing(self, repo, files): """Return count of files that actually exist in the repository directory.""" existing = 0 From 4de94bc0e3ecc65e40a16cf19dd934ac3b413023 Mon Sep 17 00:00:00 2001 From: Puneet Dixit <236133619+puneetdixit200@users.noreply.github.com> Date: Sun, 24 May 2026 00:21:37 +0530 Subject: [PATCH 1347/1392] test: accept stderr in string process adapter Match the AutoInterrupt wait signature so diff parser tests can pass preserved stderr through finalize_process. Assisted-by: OpenAI GPT-5 --- test/lib/helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/helper.py b/test/lib/helper.py index 2fc015dfa..1c110e103 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -90,7 +90,7 @@ def __init__(self, input_string): self.stdout = io.BytesIO(input_string) self.stderr = io.BytesIO() - def wait(self): + def wait(self, stderr=None): return 0 poll = wait From e3a75d0bf38616fbbf3029d540965506003531d8 Mon Sep 17 00:00:00 2001 From: Eliah Kagan Date: Sat, 23 May 2026 13:36:58 -0400 Subject: [PATCH 1348/1392] Cut xtrace noise from POSIX-ownership diagnostic steps The "Show POSIX file ownership" step in each test workflow looped over a hard-coded path list with one `ls -ld` per iteration. Bash's xtrace -- active throughout (from `~/.bash_profile` on Cygwin and from the `-x` flag in GHA's default shell line on Ubuntu / macOS / Alpine) -- reprints the entire `for` expression's expanded word list at the start of every iteration. For nine paths that's nine long traces of the same word list, drowning out the `ls -ld` output we want to read. Collapse the loop into a single multi-arg `ls -ld --`: xtrace prints the expanded command line once, `ls` produces one line per existing path and a `ls: cannot access '': No such file or directory` line per missing path. `2>&1` merges those missing-path messages into the log stream alongside the existing-path output; `|| true` keeps the step from failing under `set -e` when any path is missing. The format of missing-path reporting changes from `(missing: )` to `ls: cannot access '': No such file or directory`. Both convey the same information; the new form is slightly more verbose per missing path but eliminates the per-iteration trace reprint that dominated the original output. Cosmetic-only; no change to the diagnostic information surfaced. Flagged on PR #2154 as a follow-up: https://github.com/gitpython-developers/GitPython/pull/2154#pullrequestreview-4307636857 Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/alpine-test.yml | 8 +++----- .github/workflows/cygwin-test.yml | 8 +++----- .github/workflows/pythonpackage.yml | 8 +++----- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 4183f0e0d..5c999e487 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -63,17 +63,15 @@ jobs: - name: Show POSIX file ownership run: | - for p in \ + ls -ld -- \ "$(pwd)" \ "$(pwd)/.git" \ "$(pwd)/git/ext/gitdb" \ "$(pwd)/git/ext/gitdb/.git" \ "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ - "${HOME:?HOME is not set}/.gitconfig" - do - ls -ld -- "$p" 2>/dev/null || echo "(missing: $p)" - done + "${HOME:?HOME is not set}/.gitconfig" \ + 2>&1 || true - name: Show safe.directory entries # `actions/checkout`'s safe.directory add is only durable for the diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index c12ccb3cf..17ba4bc82 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -89,7 +89,7 @@ jobs: # `is_path_owned_by_current_user` reduces to, so this is the view that # determines whether `safe.directory` is consulted. run: | - for p in \ + ls -ld -- \ "$(pwd)" \ "$(pwd)/.git" \ "$(pwd)/git/ext/gitdb" \ @@ -98,10 +98,8 @@ jobs: "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ "$(pwd)/.git/modules/gitdb/modules/smmap" \ - "${HOME:?HOME is not set}/.gitconfig" - do - ls -ld -- "$p" 2>/dev/null || echo "(missing: $p)" - done + "${HOME:?HOME is not set}/.gitconfig" \ + 2>&1 || true - name: Show NTFS file ownership # Authoritative NTFS Owner via Get-Acl, with no Cygwin SID-to-uid layer diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index cffafc59a..6746b92c6 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -94,17 +94,15 @@ jobs: # not be informative here. The NTFS Owner check below covers Windows. if: matrix.os-type != 'windows' run: | - for p in \ + ls -ld -- \ "$(pwd)" \ "$(pwd)/.git" \ "$(pwd)/git/ext/gitdb" \ "$(pwd)/git/ext/gitdb/.git" \ "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ - "${HOME:?HOME is not set}/.gitconfig" - do - ls -ld -- "$p" 2>/dev/null || echo "(missing: $p)" - done + "${HOME:?HOME is not set}/.gitconfig" \ + 2>&1 || true - name: Show NTFS file ownership # Windows only. Reads NTFS Owner directly via Get-Acl, which is the From 5d8d15675011dde9e874a94339d1994808ffbd92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20L=C3=B6sch?= Date: Wed, 27 May 2026 16:52:19 +0200 Subject: [PATCH 1349/1392] refactor: seperate out Progress type --- git/remote.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/git/remote.py b/git/remote.py index 20e42b412..18d4829af 100644 --- a/git/remote.py +++ b/git/remote.py @@ -517,6 +517,9 @@ def iter_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> NoReturn: # -> raise NotImplementedError +Progress = Union[RemoteProgress, "UpdateProgress", Callable[..., RemoteProgress], None] + + class Remote(LazyMixin, IterableObj): """Provides easy read and write access to a git remote. @@ -872,7 +875,7 @@ def update(self, **kwargs: Any) -> "Remote": def _get_fetch_info_from_stderr( self, proc: "Git.AutoInterrupt", - progress: Union[Callable[..., Any], RemoteProgress, None], + progress: Progress, kill_after_timeout: Union[None, float] = None, ) -> IterableList["FetchInfo"]: progress = to_progress_instance(progress) @@ -1000,7 +1003,7 @@ def _assert_refspec(self) -> None: def fetch( self, refspec: Union[str, List[str], None] = None, - progress: Union[RemoteProgress, None, "UpdateProgress"] = None, + progress: Progress = None, verbose: bool = True, kill_after_timeout: Union[None, float] = None, allow_unsafe_protocols: bool = False, @@ -1081,7 +1084,7 @@ def fetch( def pull( self, refspec: Union[str, List[str], None] = None, - progress: Union[RemoteProgress, "UpdateProgress", None] = None, + progress: Progress = None, kill_after_timeout: Union[None, float] = None, allow_unsafe_protocols: bool = False, allow_unsafe_options: bool = False, @@ -1135,7 +1138,7 @@ def pull( def push( self, refspec: Union[str, List[str], None] = None, - progress: Union[RemoteProgress, "UpdateProgress", Callable[..., RemoteProgress], None] = None, + progress: Progress = None, kill_after_timeout: Union[None, float] = None, allow_unsafe_protocols: bool = False, allow_unsafe_options: bool = False, From 9d2d01ccef2f79c840828afe04698254dff354c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:49:49 +0000 Subject: [PATCH 1350/1392] Bump https://github.com/astral-sh/ruff-pre-commit Bumps the pre-commit group with 1 update: [https://github.com/astral-sh/ruff-pre-commit](https://github.com/astral-sh/ruff-pre-commit). Updates `https://github.com/astral-sh/ruff-pre-commit` from v0.15.12 to 0.15.15 - [Release notes](https://github.com/astral-sh/ruff-pre-commit/releases) - [Commits](https://github.com/astral-sh/ruff-pre-commit/compare/v0.15.12...v0.15.15) --- updated-dependencies: - dependency-name: https://github.com/astral-sh/ruff-pre-commit dependency-version: 0.15.15 dependency-type: direct:production dependency-group: pre-commit ... Signed-off-by: dependabot[bot] --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3ab67035..9cc97962d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: exclude: ^test/fixtures/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.12 + rev: v0.15.15 hooks: - id: ruff-check args: ["--fix"] From b8a4266155d106b27900e3878dd5ce7e75023e78 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 13:32:12 +0000 Subject: [PATCH 1351/1392] Bump actions/checkout from 6 to 7 Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/alpine-test.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/cygwin-test.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index 5c999e487..fede7264a 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -26,7 +26,7 @@ jobs: adduser runner docker shell: sh -exo pipefail {0} # Run this as root, not the "runner" user. - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e243416a8..373ec9339 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -47,7 +47,7 @@ jobs: # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 # Add any setup steps before running the `github/codeql-action/init` action. # This includes steps like installing compilers or runtimes (`actions/setup-node` diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 17ba4bc82..122183b76 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -34,7 +34,7 @@ jobs: git config --global core.autocrlf false # Affects the non-Cygwin git. shell: pwsh # Do this outside Cygwin, to affect actions/checkout. - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e32e946c8..005ab022d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-slim steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 6746b92c6..c01f90a17 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -50,7 +50,7 @@ jobs: shell: bash --noprofile --norc -exo pipefail {0} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 From edec9d4c66cfbbb14150d45101824747d9e6b12f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 02:22:33 +0000 Subject: [PATCH 1352/1392] Bump actions/checkout from 6 to 7 Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index c08f5b5db..cedd8d849 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -24,7 +24,7 @@ jobs: continue-on-error: ${{ matrix.experimental }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: From 2afc4980ec9ee3bb3c94670a511fd4d92d85f02c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:32:34 +0000 Subject: [PATCH 1353/1392] Bump actions/checkout from 6 to 7 Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index ca5ae25d0..ffc12ca3a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -25,7 +25,7 @@ jobs: continue-on-error: ${{ matrix.experimental }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} From 61cabcda344577b44701b3613b08d3115bbcfb1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 28 Jun 2026 13:32:09 +0000 Subject: [PATCH 1354/1392] Bump git/ext/gitdb from `0a019a2` to `4950ea9` Bumps [git/ext/gitdb](https://github.com/gitpython-developers/gitdb) from `0a019a2` to `4950ea9`. - [Release notes](https://github.com/gitpython-developers/gitdb/releases) - [Commits](https://github.com/gitpython-developers/gitdb/compare/0a019a2e2bd73158cf8b637ad78b5d4b8f15e42e...4950ea91247905d094fc4cc1e0b26282aad6afa4) --- updated-dependencies: - dependency-name: git/ext/gitdb dependency-version: 4950ea91247905d094fc4cc1e0b26282aad6afa4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- git/ext/gitdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/ext/gitdb b/git/ext/gitdb index 0a019a2e2..4950ea912 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 0a019a2e2bd73158cf8b637ad78b5d4b8f15e42e +Subproject commit 4950ea91247905d094fc4cc1e0b26282aad6afa4 From d872adbcc4477768205e65d6c48c96e008b26693 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 13:37:51 +0000 Subject: [PATCH 1355/1392] Bump https://github.com/astral-sh/ruff-pre-commit Bumps the pre-commit group with 1 update: [https://github.com/astral-sh/ruff-pre-commit](https://github.com/astral-sh/ruff-pre-commit). Updates `https://github.com/astral-sh/ruff-pre-commit` from v0.15.15 to 0.15.20 - [Release notes](https://github.com/astral-sh/ruff-pre-commit/releases) - [Commits](https://github.com/astral-sh/ruff-pre-commit/compare/v0.15.15...v0.15.20) --- updated-dependencies: - dependency-name: https://github.com/astral-sh/ruff-pre-commit dependency-version: 0.15.20 dependency-type: direct:production dependency-group: pre-commit ... Signed-off-by: dependabot[bot] --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9cc97962d..a13e85d40 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: exclude: ^test/fixtures/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.15 + rev: v0.15.20 hooks: - id: ruff-check args: ["--fix"] From 2ab051697940144545a14d2f4a639c6409c3187f Mon Sep 17 00:00:00 2001 From: abhiprd2000 <8292aniarc@gmail.com> Date: Thu, 9 Jul 2026 21:22:50 +0530 Subject: [PATCH 1356/1392] Reject abbreviated forms of unsafe git options --- git/cmd.py | 15 ++++++++++++--- test/test_clone.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 92ca09c2a..493b8456d 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -967,11 +967,20 @@ def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> arbitrary commands. These are blocked by default. """ # Options can be of the form `foo`, `--foo`, `--foo bar`, or `--foo=bar`. + # Git accepts any unambiguous prefix of a long option, so an abbreviated + # spelling such as `--upl` for `--upload-pack` must be rejected too. An + # option is unsafe if its canonical name is a prefix of any blocked + # option's canonical name. canonical_unsafe_options = {cls._canonicalize_option_name(option): option for option in unsafe_options} for option in options: - unsafe_option = canonical_unsafe_options.get(cls._canonicalize_option_name(option)) - if unsafe_option is not None: - raise UnsafeOptionError(f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it.") + candidate = cls._canonicalize_option_name(option) + if not candidate: + continue + for canonical, unsafe_option in canonical_unsafe_options.items(): + if canonical.startswith(candidate): + raise UnsafeOptionError( + f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it." + ) AutoInterrupt: TypeAlias = _AutoInterrupt diff --git a/test/test_clone.py b/test/test_clone.py index 653d50aa3..3544e74d1 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -138,6 +138,21 @@ def test_clone_unsafe_options(self, rw_repo): rw_repo.clone(tmp_dir, **unsafe_option) assert not tmp_file.exists() + @with_rw_repo("HEAD") + def test_clone_unsafe_options_abbreviated(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = pathlib.Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + f"--upl='touch {tmp_file}'", + f"--upload-pac='touch {tmp_file}'", + "--conf=protocol.ext.allow=always", + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + rw_repo.clone(tmp_dir, multi_options=[unsafe_option]) + assert not tmp_file.exists() + @with_rw_repo("HEAD") def test_clone_unsafe_options_are_checked_after_splitting_multi_options(self, rw_repo): with tempfile.TemporaryDirectory() as tdir: From 40f1424f035ae7ec6801e4622562a16064f48e55 Mon Sep 17 00:00:00 2001 From: Harshita Yadav <146022516+harshitayadavv@users.noreply.github.com> Date: Sun, 12 Jul 2026 17:20:49 +0530 Subject: [PATCH 1357/1392] Add Commit.is_shallow property; document stats() limitation at shallow boundary (#2167) Accessing .stats on a commit at the boundary of a shallow clone raises GitCommandError because the commit's parent SHA was never fetched. This adds an is_shallow property to detect this case ahead of time by checking the repository's shallow file, and documents the limitation on stats(). Co-authored-by: Claude --- AUTHORS | 1 + git/objects/commit.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/AUTHORS b/AUTHORS index 15333e1e5..95205f439 100644 --- a/AUTHORS +++ b/AUTHORS @@ -57,5 +57,6 @@ Contributors are: -Jonas Scharpf -Gordon Marx -Enji Cooper +-Harshita Yadav Portions derived from other open source works and are clearly marked. diff --git a/git/objects/commit.py b/git/objects/commit.py index da7677ee0..d98664da5 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -374,6 +374,11 @@ def stats(self) -> Stats: """Create a git stat from changes between this commit and its first parent or from all changes done if this is the very first commit. + :note: + If this commit is at the boundary of a shallow clone, this will + raise :exc:`~git.exc.GitCommandError`, since the parent object + was never fetched and only exists as a reference on this commit. + :return: :class:`Stats` """ From 181e8ede7543dbbdcee92756cea260f675316c63 Mon Sep 17 00:00:00 2001 From: "GPT 5.6" Date: Sun, 12 Jul 2026 13:45:42 +0200 Subject: [PATCH 1358/1392] Address review feedback about unsafe option abbreviations The prefix check could reject otherwise allowed one-letter short options when an unrelated blocked long option began with the same character. Keep exact matches for every spelling, but restrict prefix matching to explicit long options and multi-character kwargs. This addresses the review comment: single-letter short options and kwargs must not be treated as long-option abbreviations. It also adds coverage for abbreviated unsafe kwargs so both clone validation paths remain protected. Reviewd-by: Sebastian Thiel --- git/cmd.py | 10 +++++++++- test/test_clone.py | 10 ++++++++++ test/test_git.py | 10 ++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/git/cmd.py b/git/cmd.py index 493b8456d..080f34d1a 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -970,12 +970,20 @@ def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> # Git accepts any unambiguous prefix of a long option, so an abbreviated # spelling such as `--upl` for `--upload-pack` must be rejected too. An # option is unsafe if its canonical name is a prefix of any blocked - # option's canonical name. + # option's canonical name. Only long options and multi-character kwargs + # can be abbreviations; single-character short options remain exact-match + # only. canonical_unsafe_options = {cls._canonicalize_option_name(option): option for option in unsafe_options} + options_are_kwargs = all(not option.startswith("-") for option in options) for option in options: candidate = cls._canonicalize_option_name(option) if not candidate: continue + unsafe_option = canonical_unsafe_options.get(candidate) + if unsafe_option is not None: + raise UnsafeOptionError(f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it.") + if not (option.startswith("--") or (options_are_kwargs and len(candidate) > 1)): + continue for canonical, unsafe_option in canonical_unsafe_options.items(): if canonical.startswith(candidate): raise UnsafeOptionError( diff --git a/test/test_clone.py b/test/test_clone.py index 3544e74d1..5a447b8c4 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -153,6 +153,16 @@ def test_clone_unsafe_options_abbreviated(self, rw_repo): rw_repo.clone(tmp_dir, multi_options=[unsafe_option]) assert not tmp_file.exists() + unsafe_kwargs = [ + {"upl": f"touch {tmp_file}"}, + {"upload_pac": f"touch {tmp_file}"}, + {"conf": "protocol.ext.allow=always"}, + ] + for unsafe_option in unsafe_kwargs: + with self.assertRaises(UnsafeOptionError): + rw_repo.clone(tmp_dir, **unsafe_option) + assert not tmp_file.exists() + @with_rw_repo("HEAD") def test_clone_unsafe_options_are_checked_after_splitting_multi_options(self, rw_repo): with tempfile.TemporaryDirectory() as tdir: diff --git a/test/test_git.py b/test/test_git.py index 24b60af9d..d46907d69 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -170,6 +170,16 @@ def test_check_unsafe_options_normalizes_kwargs(self): with self.assertRaises(UnsafeOptionError): Git.check_unsafe_options(options=options, unsafe_options=unsafe_options) + def test_check_unsafe_options_does_not_treat_short_options_as_abbreviations(self): + unsafe_options = ["--upload-pack"] + + Git.check_unsafe_options(options=["-u", "u", "-upl"], unsafe_options=unsafe_options) + with self.assertRaises(UnsafeOptionError): + Git.check_unsafe_options(options=["--u"], unsafe_options=unsafe_options) + + def test_check_unsafe_options_does_not_treat_option_values_as_abbreviations(self): + Git.check_unsafe_options(options=["--branch", "conf"], unsafe_options=["--config"]) + _shell_cases = ( # value_in_call, value_from_class, expected_popen_arg (None, False, False), From 56806080c1348749b07daa4a2024ce47b3cad285 Mon Sep 17 00:00:00 2001 From: "GPT 5.6" Date: Sun, 12 Jul 2026 14:12:16 +0200 Subject: [PATCH 1359/1392] fix: reject joined unsafe short options GitPython blocks options such as --upload-pack/-u and --config/-c because they can execute arbitrary commands unless unsafe options are explicitly allowed. Git also accepts a short option with its value joined to the same token, including after clusterable flags. Forms such as -uVALUE, -fuVALUE, -cVALUE, and -vcVALUE could therefore bypass an exact-token check. Parse single-dash option tokens sufficiently to recognize blocked short options while preserving safe attached values such as -oupstream. Also distinguish clone multi-option values from bare kwargs so positional values are not treated as long-option abbreviations. This completes the option-validation hardening for GHSA-2f96-g7mh-g2hx. Reviewed-by: Sebastian Thiel --- git/cmd.py | 40 ++++++++++++++++++++++++++++++++++++---- test/test_clone.py | 12 ++++++++++++ test/test_git.py | 18 ++++++++++++++++++ test/test_remote.py | 14 ++++++++++++-- 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 080f34d1a..6462aba32 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -961,10 +961,23 @@ def _canonicalize_option_name(cls, option: str) -> str: @classmethod def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> None: - """Check for unsafe options. - - Some options that are passed to ``git `` can be used to execute - arbitrary commands. These are blocked by default. + """Raise :class:`~git.exc.UnsafeOptionError` for blocked option spellings. + + In addition to exact matches, this rejects abbreviated long options accepted + by Git (for example, ``--upl`` for ``--upload-pack``) and unsafe short options + whose values are joined to the same token, including after clusterable flags + (for example, ``-uVALUE`` and ``-fuVALUE``). + + A list containing only bare names is treated as normalized keyword arguments, + so multi-character names such as ``upload_p`` are checked as long-option + abbreviations. If any item starts with ``-``, the list is treated as tokenized + command-line input: bare items can be option values and are not checked as + abbreviations. Thus ``["--origin", "upload"]`` is allowed. Single-dash options + use short-option parsing rather than broad prefix matching, preserving safe + attached values such as ``-oupstream`` and ``-bcurrent``. + + Some options passed to ``git `` can execute arbitrary commands and + are therefore blocked by default unless the caller explicitly allows them. """ # Options can be of the form `foo`, `--foo`, `--foo bar`, or `--foo=bar`. # Git accepts any unambiguous prefix of a long option, so an abbreviated @@ -974,6 +987,15 @@ def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> # can be abbreviations; single-character short options remain exact-match # only. canonical_unsafe_options = {cls._canonicalize_option_name(option): option for option in unsafe_options} + unsafe_short_options = { + canonical: option + for canonical, option in canonical_unsafe_options.items() + if option.startswith("-") and not option.startswith("--") and len(canonical) == 1 + } + # These value-less Git flags can be clustered before another short option + # (for example, ``-fuVALUE``). Stop at any other character because it may + # begin an attached value, as ``o`` does in the safe option ``-oupstream``. + clusterable_short_options = frozenset("46flnqsv") options_are_kwargs = all(not option.startswith("-") for option in options) for option in options: candidate = cls._canonicalize_option_name(option) @@ -982,6 +1004,16 @@ def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> unsafe_option = canonical_unsafe_options.get(candidate) if unsafe_option is not None: raise UnsafeOptionError(f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it.") + option_token = option.split("=", 1)[0].split(None, 1)[0] + if option_token.startswith("-") and not option_token.startswith("--"): + for option_char in option_token[1:]: + unsafe_option = unsafe_short_options.get(option_char) + if unsafe_option is not None: + raise UnsafeOptionError( + f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it." + ) + if option_char not in clusterable_short_options: + break if not (option.startswith("--") or (options_are_kwargs and len(candidate) > 1)): continue for canonical, unsafe_option in canonical_unsafe_options.items(): diff --git a/test/test_clone.py b/test/test_clone.py index 5a447b8c4..81c08a8df 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -118,8 +118,13 @@ def test_clone_unsafe_options(self, rw_repo): unsafe_options = [ f"--upload-pack='touch {tmp_file}'", f"-u 'touch {tmp_file}'", + f"-utouch {tmp_file}; false", + f"-futouch${{IFS}}{tmp_file}; false", + f"-qutouch${{IFS}}{tmp_file}; false", "--config=protocol.ext.allow=always", "-c protocol.ext.allow=always", + "-cprotocol.ext.allow=always", + "-vcprotocol.ext.allow=always", ] for unsafe_option in unsafe_options: with self.assertRaises(UnsafeOptionError): @@ -216,7 +221,9 @@ def test_clone_safe_options(self, rw_repo): options = [ "--depth=1", "--single-branch", + "--origin upload", "-q", + "-oupstream", ] for option in options: destination = tmp_dir / option @@ -232,8 +239,13 @@ def test_clone_from_unsafe_options(self, rw_repo): unsafe_options = [ f"--upload-pack='touch {tmp_file}'", f"-u 'touch {tmp_file}'", + f"-utouch {tmp_file}; false", + f"-futouch${{IFS}}{tmp_file}; false", + f"-qutouch${{IFS}}{tmp_file}; false", "--config=protocol.ext.allow=always", "-c protocol.ext.allow=always", + "-cprotocol.ext.allow=always", + "-vcprotocol.ext.allow=always", ] for unsafe_option in unsafe_options: with self.assertRaises(UnsafeOptionError): diff --git a/test/test_git.py b/test/test_git.py index d46907d69..bfee71297 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -180,6 +180,24 @@ def test_check_unsafe_options_does_not_treat_short_options_as_abbreviations(self def test_check_unsafe_options_does_not_treat_option_values_as_abbreviations(self): Git.check_unsafe_options(options=["--branch", "conf"], unsafe_options=["--config"]) + def test_check_unsafe_options_rejects_joined_unsafe_short_options(self): + cases = [ + (["-utouch /tmp/pwn"], ["-u"]), + (["-futouch /tmp/pwn"], ["-u"]), + (["-qutouch${IFS}/tmp/pwn"], ["-u"]), + (["-cprotocol.ext.allow=always"], ["-c"]), + (["-vcprotocol.ext.allow=always"], ["-c"]), + ] + + for options, unsafe_options in cases: + with self.assertRaises(UnsafeOptionError): + Git.check_unsafe_options(options=options, unsafe_options=unsafe_options) + + def test_check_unsafe_options_allows_attached_safe_short_option_values(self): + unsafe_options = ["--upload-pack", "-u", "--config", "-c"] + + Git.check_unsafe_options(options=["-oupstream", "-bcurrent"], unsafe_options=unsafe_options) + _shell_cases = ( # value_in_call, value_from_class, expected_popen_arg (None, False, False), diff --git a/test/test_remote.py b/test/test_remote.py index 2230c8df4..81446aec2 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -832,7 +832,11 @@ def test_fetch_unsafe_options(self, rw_repo): remote = rw_repo.remote("origin") tmp_dir = Path(tdir) tmp_file = tmp_dir / "pwn" - unsafe_options = [{"upload-pack": f"touch {tmp_file}"}, {"upload_pack": f"touch {tmp_file}"}] + unsafe_options = [ + {"upload-pack": f"touch {tmp_file}"}, + {"upload_pack": f"touch {tmp_file}"}, + {"upload_p": f"touch {tmp_file}"}, + ] for unsafe_option in unsafe_options: with self.assertRaises(UnsafeOptionError): remote.fetch(**unsafe_option) @@ -900,7 +904,11 @@ def test_pull_unsafe_options(self, rw_repo): remote = rw_repo.remote("origin") tmp_dir = Path(tdir) tmp_file = tmp_dir / "pwn" - unsafe_options = [{"upload-pack": f"touch {tmp_file}"}, {"upload_pack": f"touch {tmp_file}"}] + unsafe_options = [ + {"upload-pack": f"touch {tmp_file}"}, + {"upload_pack": f"touch {tmp_file}"}, + {"upload_p": f"touch {tmp_file}"}, + ] for unsafe_option in unsafe_options: with self.assertRaises(UnsafeOptionError): remote.pull(**unsafe_option) @@ -971,7 +979,9 @@ def test_push_unsafe_options(self, rw_repo): unsafe_options = [ {"receive-pack": f"touch {tmp_file}"}, {"receive_pack": f"touch {tmp_file}"}, + {"receive_p": f"touch {tmp_file}"}, {"exec": f"touch {tmp_file}"}, + {"exe": f"touch {tmp_file}"}, ] for unsafe_option in unsafe_options: assert not tmp_file.exists() From 1551238cb52c78755632264fa0aa0bc750f89b0f Mon Sep 17 00:00:00 2001 From: Codex GPT-5 Date: Sun, 12 Jul 2026 14:35:09 +0200 Subject: [PATCH 1360/1392] fix: allow relative config paths with includes (#2103) GitConfigParser asserted while resolving a relative include whenever the top-level configuration file had itself been supplied as a relative path. A regression test constructs that combination and verifies the included value is available. Normalize path-based inputs to absolute paths before reading them. This lets relative includes resolve against the containing file and ensures cycle detection uses the same canonical spelling for top-level and included paths. This matches Git's documented include behavior and the relative and chained-relative coverage in t/t1305-config-include.sh. Validation: - uv run pytest -q test/test_config.py - uv run pre-commit run --files git/config.py test/test_config.py - uv run mypy git/config.py - git diff --check - full pytest: 631 passed, 75 skipped, 34 unrelated environment failures (uninitialized nested submodules and missing legacy master branch) --- git/config.py | 2 ++ test/test_config.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/git/config.py b/git/config.py index 82747eadd..64f501424 100644 --- a/git/config.py +++ b/git/config.py @@ -634,6 +634,8 @@ def read(self) -> None: # type: ignore[override] files_to_read = list(self._file_or_files) # END ensure we have a copy of the paths to handle + files_to_read = [osp.abspath(path) if isinstance(path, (str, os.PathLike)) else path for path in files_to_read] + seen = set(files_to_read) num_read_include_files = 0 while files_to_read: diff --git a/test/test_config.py b/test/test_config.py index 3ddaf0a4b..897caba4c 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -310,6 +310,20 @@ def check_test_value(cr, value): with GitConfigParser(fpa, read_only=True) as cr: check_test_value(cr, tv) + @with_rw_directory + def test_config_relative_path_include(self, rw_dir): + included_path = osp.join(rw_dir, "included") + with GitConfigParser(included_path, read_only=False) as cw: + cw.set_value("included", "value", "included") + + config_path = osp.join(rw_dir, "config") + with GitConfigParser(config_path, read_only=False) as cw: + cw.set_value("include", "path", "included") + + relative_config_path = osp.relpath(config_path) + with GitConfigParser(relative_config_path, read_only=True) as cr: + assert cr.get_value("included", "value") == "included" + @with_rw_directory def test_multiple_include_paths_with_same_key(self, rw_dir): """Test that multiple 'path' entries under [include] are all respected. From f03301779d1eca32825973784c8b64c4a28a5226 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 12 Jul 2026 14:48:47 +0200 Subject: [PATCH 1361/1392] chore: avoid duplicate runs of CI in PRs Run once for PRs, then again after merge. --- .github/workflows/alpine-test.yml | 6 +++++- .github/workflows/codeql.yml | 1 + .github/workflows/cygwin-test.yml | 6 +++++- .github/workflows/lint.yml | 6 +++++- .github/workflows/pythonpackage.yml | 6 +++++- git/ext/gitdb | 2 +- 6 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index fede7264a..a5b63f423 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -1,6 +1,10 @@ name: test-alpine -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [main] + pull_request: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 373ec9339..1ad1a612c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,6 +13,7 @@ name: "CodeQL" on: push: + branches: [main] pull_request: schedule: - cron: '27 10 * * 3' diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 122183b76..6f9f347a9 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -1,6 +1,10 @@ name: test-cygwin -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [main] + pull_request: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 005ab022d..79d786669 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,6 +1,10 @@ name: Lint -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [main] + pull_request: + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index c01f90a17..fe3e26236 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -3,7 +3,11 @@ name: Python package -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [main] + pull_request: + workflow_dispatch: permissions: contents: read diff --git a/git/ext/gitdb b/git/ext/gitdb index 4950ea912..335c0f661 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 4950ea91247905d094fc4cc1e0b26282aad6afa4 +Subproject commit 335c0f66173eecdc7b2597c2b6c3d1fde795df30 From 701ce32fe5ba8cb622c0e0342a376a6beb47d738 Mon Sep 17 00:00:00 2001 From: "GPT 5.6" Date: Sun, 12 Jul 2026 15:06:42 +0200 Subject: [PATCH 1362/1392] fix: Guard unsafe git command options (GHSA-956x-8gvw-wg5v) Reject command options that can execute arbitrary helpers or write files when passed through archive, revision traversal, blame, clone, fetch, pull, push, and ls-remote APIs. Validate positional option tokens and transformed keyword options consistently, including Git-supported long-option abbreviations, attached short-option values, and falsey stringifiable remotes. Preserve explicit allow_unsafe_options and allow_unsafe_protocols escape hatches. Add regression coverage for each guarded API, option spelling and abbreviation forms, inert keyword values, mixed positional and keyword candidates, tuple blame options, stringifiable remotes, and unsafe commands that must not create marker files. Reviewed-by: Sebastian Thiel --- git/cmd.py | 37 +++++++++++++++++++- git/objects/commit.py | 12 +++++++ git/remote.py | 15 ++++++-- git/repo/base.py | 80 +++++++++++++++++++++++++++++++++++++++---- test/test_clone.py | 4 +++ test/test_commit.py | 13 +++++++ test/test_git.py | 16 +++++++++ test/test_remote.py | 43 ++++++++++++++++++++++- test/test_repo.py | 71 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 279 insertions(+), 12 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 6462aba32..2a4f36024 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -649,6 +649,11 @@ class Git(metaclass=_GitMeta): re_unsafe_protocol = re.compile(r"(.+)::.+") + unsafe_git_ls_remote_options = [ + # This option allows arbitrary command execution in git-ls-remote. + "--upload-pack", + ] + def __getstate__(self) -> Dict[str, Any]: return slots_to_dict(self, exclude=self._excluded_) @@ -1022,6 +1027,20 @@ def check_unsafe_options(cls, options: List[str], unsafe_options: List[str]) -> f"{unsafe_option} is not allowed, use `allow_unsafe_options=True` to allow it." ) + @classmethod + def _option_candidates(cls, args: Sequence[Any] = (), kwargs: Optional[Mapping[str, Any]] = None) -> List[str]: + """Collect possible option spellings before command-line transformation.""" + options = [ + option for option in cls._unpack_args([arg for arg in args if arg is not None]) if option.startswith("-") + ] + if kwargs: + for key, value in kwargs.items(): + values = value if isinstance(value, (list, tuple)) else (value,) + if any(value is True or (value is not False and value is not None) for value in values): + key = str(key) + options.append(f"-{key}" if len(key) == 1 else f"--{dashify(key)}") + return options + AutoInterrupt: TypeAlias = _AutoInterrupt CatFileContentStream: TypeAlias = _CatFileContentStream @@ -1079,6 +1098,22 @@ def set_persistent_git_options(self, **kwargs: Any) -> None: self._persistent_git_options = self.transform_kwargs(split_single_char_options=True, **kwargs) + def ls_remote( + self, + *args: Any, + allow_unsafe_options: bool = False, + **kwargs: Any, + ) -> Union[str, bytes, Tuple[int, Union[str, bytes], str], "Git.AutoInterrupt"]: + """List references in a remote repository. + + :param allow_unsafe_options: + Allow unsafe options, like ``--upload-pack``. + """ + if not allow_unsafe_options: + candidate_options = self._option_candidates(args, kwargs) + Git.check_unsafe_options(options=candidate_options, unsafe_options=self.unsafe_git_ls_remote_options) + return self._call_process("ls_remote", *args, **kwargs) + @property def working_dir(self) -> Union[None, PathLike]: """:return: Git directory we are working on""" @@ -1585,7 +1620,7 @@ def transform_kwargs(self, split_single_char_options: bool = True, **kwargs: Any return args @classmethod - def _unpack_args(cls, arg_list: Sequence[str]) -> List[str]: + def _unpack_args(cls, arg_list: Sequence[Any]) -> List[str]: outlist = [] if isinstance(arg_list, (list, tuple)): for arg in arg_list: diff --git a/git/objects/commit.py b/git/objects/commit.py index d98664da5..1d8e8f071 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -86,6 +86,12 @@ class Commit(base.Object, TraversableIterableObj, Diffable, Serializable): # INVARIANTS default_encoding = "UTF-8" + # Options to :manpage:`git-rev-list(1)` that can overwrite files. + unsafe_git_rev_options = [ + "--output", + "-o", + ] + type: Literal["commit"] = "commit" __slots__ = ( @@ -302,6 +308,7 @@ def iter_items( repo: "Repo", rev: Union[str, "Commit", "SymbolicReference"], paths: Union[PathLike, Sequence[PathLike]] = "", + allow_unsafe_options: bool = False, **kwargs: Any, ) -> Iterator["Commit"]: R"""Find all commits matching the given criteria. @@ -330,6 +337,11 @@ def iter_items( raise ValueError("--pretty cannot be used as parsing expects single sha's only") # END handle pretty + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([rev], kwargs), unsafe_options=cls.unsafe_git_rev_options + ) + # Use -- in all cases, to prevent possibility of ambiguous arguments. # See https://github.com/gitpython-developers/GitPython/issues/264. diff --git a/git/remote.py b/git/remote.py index 18d4829af..0c3dbfe15 100644 --- a/git/remote.py +++ b/git/remote.py @@ -1071,7 +1071,10 @@ def fetch( Git.check_unsafe_protocols(ref) if not allow_unsafe_options: - Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_fetch_options) + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=self.unsafe_git_fetch_options, + ) proc = self.repo.git.fetch( "--", self, *args, as_process=True, with_stdout=False, universal_newlines=True, v=verbose, **kwargs @@ -1125,7 +1128,10 @@ def pull( Git.check_unsafe_protocols(ref) if not allow_unsafe_options: - Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_pull_options) + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=self.unsafe_git_pull_options, + ) proc = self.repo.git.pull( "--", self, refspec, with_stdout=False, as_process=True, universal_newlines=True, v=True, **kwargs @@ -1198,7 +1204,10 @@ def push( Git.check_unsafe_protocols(ref) if not allow_unsafe_options: - Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_push_options) + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=self.unsafe_git_push_options, + ) proc = self.repo.git.push( "--", diff --git a/git/repo/base.py b/git/repo/base.py index 2d3cf24f0..913984975 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -161,6 +161,20 @@ class Repo: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---configltkeygtltvaluegt """ + unsafe_git_archive_options = [ + # Allows arbitrary command execution through the remote git-upload-archive command. + "--exec", + # Writes output to a caller-controlled filesystem path. + "--output", + "-o", + ] + + unsafe_git_revision_options = [ + # This option allows output to be written to arbitrary files before revision parsing. + "--output", + "-o", + ] + # Invariants config_level: ConfigLevels_Tup = ("system", "user", "global", "repository") """Represents the configuration level of a configuration file.""" @@ -775,6 +789,7 @@ def iter_commits( self, rev: Union[str, Commit, "SymbolicReference", None] = None, paths: Union[PathLike, Sequence[PathLike]] = "", + allow_unsafe_options: bool = False, **kwargs: Any, ) -> Iterator[Commit]: """An iterator of :class:`~git.objects.commit.Commit` objects representing the @@ -792,6 +807,9 @@ def iter_commits( Arguments to be passed to :manpage:`git-rev-list(1)`. Common ones are ``max_count`` and ``skip``. + :param allow_unsafe_options: + Allow unsafe options in the revision argument, like ``--output``. + :note: To receive only commits between two named revisions, use the ``"revA...revB"`` revision specifier. @@ -802,7 +820,18 @@ def iter_commits( if rev is None: rev = self.head.commit - return Commit.iter_items(self, rev, paths, **kwargs) + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([rev], kwargs), unsafe_options=self.unsafe_git_revision_options + ) + + return Commit.iter_items( + self, + rev, + paths, + allow_unsafe_options=allow_unsafe_options, + **kwargs, + ) def merge_base(self, *rev: TBD, **kwargs: Any) -> List[Commit]: R"""Find the closest common ancestor for the given revision @@ -1079,7 +1108,9 @@ def active_branch(self) -> Head: ) return active_branch - def blame_incremental(self, rev: str | HEAD | None, file: str, **kwargs: Any) -> Iterator["BlameEntry"]: + def blame_incremental( + self, rev: str | HEAD | None, file: str, allow_unsafe_options: bool = False, **kwargs: Any + ) -> Iterator["BlameEntry"]: """Iterator for blame information for the given file at the given revision. Unlike :meth:`blame`, this does not return the actual file's contents, only a @@ -1090,6 +1121,9 @@ def blame_incremental(self, rev: str | HEAD | None, file: str, **kwargs: Any) -> uncommitted changes. Otherwise, anything successfully parsed by :manpage:`git-rev-parse(1)` is a valid option. + :param allow_unsafe_options: + Allow unsafe options in revision argument, like ``--output``. + :return: Lazy iterator of :class:`BlameEntry` tuples, where the commit indicates the commit to blame for the line, and range indicates a span of line numbers in @@ -1098,6 +1132,10 @@ def blame_incremental(self, rev: str | HEAD | None, file: str, **kwargs: Any) -> If you combine all line number ranges outputted by this command, you should get a continuous range spanning all line numbers in the file. """ + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([rev], kwargs), unsafe_options=self.unsafe_git_revision_options + ) data: bytes = self.git.blame(rev, "--", file, p=True, incremental=True, stdout_as_string=False, **kwargs) commits: Dict[bytes, Commit] = {} @@ -1176,7 +1214,8 @@ def blame( rev: Union[str, HEAD, None], file: str, incremental: bool = False, - rev_opts: Optional[List[str]] = None, + rev_opts: Optional[Sequence[str]] = None, + allow_unsafe_options: bool = False, **kwargs: Any, ) -> List[List[Commit | List[str | bytes] | None]] | Iterator[BlameEntry] | None: """The blame information for the given file at the given revision. @@ -1186,6 +1225,9 @@ def blame( uncommitted changes. Otherwise, anything successfully parsed by :manpage:`git-rev-parse(1)` is a valid option. + :param allow_unsafe_options: + Allow unsafe options in revision argument, like ``--output``. + :return: list: [git.Commit, list: []] @@ -1195,9 +1237,14 @@ def blame( appearance. """ if incremental: - return self.blame_incremental(rev, file, **kwargs) - rev_opts = rev_opts or [] - data: bytes = self.git.blame(rev, *rev_opts, "--", file, p=True, stdout_as_string=False, **kwargs) + return self.blame_incremental(rev, file, allow_unsafe_options=allow_unsafe_options, **kwargs) + rev_opts_list = list(rev_opts or []) + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([rev, rev_opts_list], kwargs), + unsafe_options=self.unsafe_git_revision_options, + ) + data: bytes = self.git.blame(rev, *rev_opts_list, "--", file, p=True, stdout_as_string=False, **kwargs) commits: Dict[str, Commit] = {} blames: List[List[Commit | List[str | bytes] | None]] = [] @@ -1408,7 +1455,10 @@ def _clone( if not allow_unsafe_protocols: Git.check_unsafe_protocols(url) if not allow_unsafe_options: - Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=cls.unsafe_git_clone_options) + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=cls.unsafe_git_clone_options, + ) if not allow_unsafe_options and multi: Git.check_unsafe_options(options=multi, unsafe_options=cls.unsafe_git_clone_options) @@ -1583,6 +1633,8 @@ def archive( ostream: Union[TextIO, BinaryIO], treeish: Optional[str] = None, prefix: Optional[str] = None, + allow_unsafe_options: bool = False, + allow_unsafe_protocols: bool = False, **kwargs: Any, ) -> Repo: """Archive the tree at the given revision. @@ -1605,6 +1657,12 @@ def archive( repository-relative path to a directory or file to place into the archive, or a list or tuple of multiple paths. + :param allow_unsafe_options: + Allow unsafe options, like ``--exec`` or ``--output``. + + :param allow_unsafe_protocols: + Allow unsafe protocols to be used in ``remote``, like ``ext``. + :raise git.exc.GitCommandError: If something went wrong. @@ -1615,6 +1673,14 @@ def archive( treeish = self.head.commit if prefix and "prefix" not in kwargs: kwargs["prefix"] = prefix + remote = kwargs.get("remote") + if not allow_unsafe_protocols and remote is not None: + Git.check_unsafe_protocols(str(remote)) + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), + unsafe_options=self.unsafe_git_archive_options, + ) kwargs["output_stream"] = ostream path = kwargs.pop("path", []) path = cast(Union[PathLike, List[PathLike], Tuple[PathLike, ...]], path) diff --git a/test/test_clone.py b/test/test_clone.py index 81c08a8df..79d63dfdc 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -117,11 +117,13 @@ def test_clone_unsafe_options(self, rw_repo): tmp_file = tmp_dir / "pwn" unsafe_options = [ f"--upload-pack='touch {tmp_file}'", + f"--upl='touch {tmp_file}'", f"-u 'touch {tmp_file}'", f"-utouch {tmp_file}; false", f"-futouch${{IFS}}{tmp_file}; false", f"-qutouch${{IFS}}{tmp_file}; false", "--config=protocol.ext.allow=always", + "--conf=protocol.ext.allow=always", "-c protocol.ext.allow=always", "-cprotocol.ext.allow=always", "-vcprotocol.ext.allow=always", @@ -134,8 +136,10 @@ def test_clone_unsafe_options(self, rw_repo): unsafe_options = [ {"upload-pack": f"touch {tmp_file}"}, {"upload_pack": f"touch {tmp_file}"}, + {"upl": f"touch {tmp_file}"}, {"u": f"touch {tmp_file}"}, {"config": "protocol.ext.allow=always"}, + {"conf": "protocol.ext.allow=always"}, {"c": "protocol.ext.allow=always"}, ] for unsafe_option in unsafe_options: diff --git a/test/test_commit.py b/test/test_commit.py index b56ad3a18..74b7078f5 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -6,6 +6,7 @@ import copy from datetime import datetime from io import BytesIO +import tempfile import os.path as osp import re import sys @@ -15,6 +16,7 @@ from gitdb import IStream from git import Actor, Commit, Repo +from git.exc import UnsafeOptionError from git.objects.util import tzoffset, utc from git.repo.fun import touch @@ -288,6 +290,17 @@ def test_iter_items(self): # pretty not allowed. self.assertRaises(ValueError, Commit.iter_items, self.rorepo, "master", pretty="raw") + def test_iter_items_rejects_unsafe_revision(self): + with tempfile.TemporaryDirectory() as tdir: + marker = osp.join(tdir, "pwn") + self.assertRaises(UnsafeOptionError, Commit.iter_items, self.rorepo, f"--output={marker}") + + def test_iter_items_rejects_unsafe_options(self): + with tempfile.TemporaryDirectory() as tdir: + marker = osp.join(tdir, "pwn") + with self.assertRaises(UnsafeOptionError): + list(Commit.iter_items(self.rorepo, "HEAD", output=marker)) + def test_rev_list_bisect_all(self): """ 'git rev-list --bisect-all' returns additional information diff --git a/test/test_git.py b/test/test_git.py index bfee71297..e95992ba8 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -164,6 +164,9 @@ def test_check_unsafe_options_normalizes_kwargs(self): (["c"], ["-c"]), (["--upload-pack=/tmp/helper"], ["--upload-pack"]), (["--config core.filemode=false"], ["--config"]), + (["--upl=/tmp/helper"], ["--upload-pack"]), + (["conf"], ["--config"]), + (["--out=/tmp/output"], ["--output"]), ] for options, unsafe_options in cases: @@ -198,6 +201,19 @@ def test_check_unsafe_options_allows_attached_safe_short_option_values(self): Git.check_unsafe_options(options=["-oupstream", "-bcurrent"], unsafe_options=unsafe_options) + def test_option_candidates_ignore_untransformed_kwargs(self): + options = Git._option_candidates( + kwargs={ + "output": None, + "upload_pack": False, + "exec": [], + "config": [None, False], + "max_count": 1, + } + ) + + self.assertEqual(options, ["--max-count"]) + _shell_cases = ( # value_in_call, value_from_class, expected_popen_arg (None, False, False), diff --git a/test/test_remote.py b/test/test_remote.py index 81446aec2..64c41290e 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -9,7 +9,7 @@ import random import sys import tempfile -from unittest import skipIf +from unittest import mock, skipIf import pytest @@ -1017,6 +1017,47 @@ def test_push_unsafe_options_allowed(self, rw_repo): assert tmp_file.exists() tmp_file.unlink() + @with_rw_repo("HEAD") + def test_ls_remote_unsafe_options(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [ + {"upload-pack": f"touch {tmp_file}"}, + {"upload_pack": f"touch {tmp_file}"}, + {"upl": f"touch {tmp_file}"}, + ] + for unsafe_option in unsafe_options: + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote(".", **unsafe_option) + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote([f"--upload-pack={tmp_file}"], ".") + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote([f"--upl={tmp_file}"], ".") + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote(f"--upload-pack={tmp_file}", ".") + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote(f"--upl={tmp_file}", ".") + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote("--upload-pack", "touch", ".") + with self.assertRaises(UnsafeOptionError): + rw_repo.git.ls_remote("--refs", ".", upl=f"touch {tmp_file}") + + def test_ls_remote_allows_operand_named_like_unsafe_option(self): + with mock.patch.object(Git, "_call_process") as git: + Git().ls_remote("upload-pack") + git.assert_called_once() + + @with_rw_repo("HEAD") + def test_ls_remote_unsafe_options_allowed(self, rw_repo): + with tempfile.TemporaryDirectory() as tdir: + tmp_dir = Path(tdir) + tmp_file = tmp_dir / "pwn" + unsafe_options = [{"upload-pack": f"touch {tmp_file}"}] + for unsafe_option in unsafe_options: + with self.assertRaises(GitCommandError): + rw_repo.git.ls_remote(".", **unsafe_option, allow_unsafe_options=True) + @with_rw_and_rw_remote_repo("0.1.6") def test_fetch_unsafe_branch_name(self, rw_repo, remote_repo): # Create branch with a name containing a NBSP diff --git a/test/test_repo.py b/test/test_repo.py index d2dd1ea5d..cfad73489 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -37,6 +37,8 @@ Submodule, Tree, ) +from git.exc import UnsafeOptionError +from git.exc import UnsafeProtocolError from git.exc import BadObject from git.repo.fun import touch from git.util import bin_to_hex, cwd, cygpath, join_path_native, rmfile, rmtree @@ -422,6 +424,54 @@ def test_archive(self): assert stream.tell() os.remove(stream.name) # Do it this way so we can inspect the file on failure. + def test_archive_rejects_unsafe_options(self): + with tempfile.TemporaryDirectory() as tdir: + output_marker = osp.join(tdir, "pwn") + with self.assertRaises(UnsafeOptionError): + self.rorepo.archive(io.BytesIO(), "0.1.6", exec=f"touch {output_marker}") + assert not osp.exists(output_marker) + with self.assertRaises(UnsafeOptionError): + self.rorepo.archive(io.BytesIO(), "0.1.6", output=output_marker) + assert not osp.exists(output_marker) + + def test_archive_rejects_unsafe_remote_protocol(self): + with tempfile.TemporaryDirectory() as tdir: + output_marker = osp.join(tdir, "pwn") + with self.assertRaises(UnsafeProtocolError): + self.rorepo.archive(io.BytesIO(), "HEAD", remote=f"ext::sh -c touch% {output_marker}") + assert not osp.exists(output_marker) + + def test_archive_preserves_positional_allow_unsafe_options(self): + with mock.patch.object(Git, "_call_process") as git: + self.rorepo.archive(io.BytesIO(), "HEAD", None, True, exec="git-upload-archive") + git.assert_called_once() + + def test_archive_accepts_stringifiable_remote(self): + class StringifiableRemote: + def __str__(self): + return "origin" + + with mock.patch.object(Git, "_call_process") as git: + self.rorepo.archive(io.BytesIO(), "HEAD", remote=StringifiableRemote()) + git.assert_called_once() + + def test_archive_rejects_unsafe_falsey_remote_protocol(self): + class FalseyRemote: + def __bool__(self): + return False + + def __str__(self): + return "ext::sh -c true" + + with self.assertRaises(UnsafeProtocolError): + self.rorepo.archive(io.BytesIO(), "HEAD", remote=FalseyRemote()) + + def test_iter_commits_rejects_unsafe_revision(self): + with tempfile.TemporaryDirectory() as tdir: + target = osp.join(tdir, "pwn") + with self.assertRaises(UnsafeOptionError): + list(self.rorepo.iter_commits(f"--output={target}", max_count=1)) + @mock.patch.object(Git, "_call_process") def test_should_display_blame_information(self, git): git.return_value = fixture("blame") @@ -471,6 +521,27 @@ def test_blame_real(self): assert c, "Should have executed at least one blame command" assert nml, "There should at least be one blame commit that contains multiple lines" + def test_blame_rejects_unsafe_revision(self): + with tempfile.TemporaryDirectory() as tdir: + output_marker = osp.join(tdir, "pwn") + with self.assertRaises(UnsafeOptionError): + self.rorepo.blame(f"--output={output_marker}", "README.md") + assert not osp.exists(output_marker) + + def test_blame_rejects_unsafe_options(self): + with tempfile.TemporaryDirectory() as tdir: + output_marker = osp.join(tdir, "pwn") + with self.assertRaises(UnsafeOptionError): + self.rorepo.blame("HEAD", "README.md", output=output_marker) + assert not osp.exists(output_marker) + + def test_blame_rejects_unsafe_rev_opts(self): + with tempfile.TemporaryDirectory() as tdir: + output_marker = osp.join(tdir, "pwn") + with self.assertRaises(UnsafeOptionError): + self.rorepo.blame("HEAD", "README.md", rev_opts=(f"--output={output_marker}",)) + assert not osp.exists(output_marker) + @mock.patch.object(Git, "_call_process") def test_blame_incremental(self, git): # Loop over two fixtures, create a test fixture for 2.11.1+ syntax. From 7b0764d309334b0f27f503d7fa1a395cbd165de0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 12 Jul 2026 15:38:18 +0200 Subject: [PATCH 1363/1392] bump to v3.1.51 --- VERSION | 2 +- doc/source/changes.rst | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0bc461141..9e5b44197 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.50 +3.1.51 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index b5152b3c5..8337478dd 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,23 @@ Changelog ========= +3.1.51 +====== + +This is primarily a security release. It prevents additional argument-injection +paths that could allow execution of arbitrary commands or writing to arbitrary +files through unsafe Git options (GHSA-956x-8gvw-wg5v), and closes bypasses of +the existing protections using abbreviated long options or joined short options +(GHSA-2f96-g7mh-g2hx). + +The release also improves support for relative worktree paths and diffs against +the empty tree, preserves stderr from failed diff processes, supports relative +configuration include paths, and includes assorted documentation, typing, test, +and CI improvements. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.51 + 3.1.50 ====== From 67082d64d53f4d9e3bbf270173f92799e19e31f7 Mon Sep 17 00:00:00 2001 From: Codex GPT-5 Date: Sun, 12 Jul 2026 15:58:55 +0200 Subject: [PATCH 1364/1392] test: skip relative config include across Windows drives On Windows, the writable temporary directory and the checkout can be on different drives. os.path.relpath() raises ValueError for such paths before GitConfigParser is exercised. Skip this relative-path-specific test on Windows, where the test setup cannot reliably construct the required relative path. Validation: - pytest -q test/test_config.py - ruff check test/test_config.py - ruff format --check test/test_config.py --- test/test_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_config.py b/test/test_config.py index 897caba4c..c4e39db48 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -320,6 +320,9 @@ def test_config_relative_path_include(self, rw_dir): with GitConfigParser(config_path, read_only=False) as cw: cw.set_value("include", "path", "included") + if osp.splitdrive(config_path)[0] != osp.splitdrive(os.getcwd())[0]: + pytest.skip("The temporary directory and checkout are on different drives") + relative_config_path = osp.relpath(config_path) with GitConfigParser(relative_config_path, read_only=True) as cr: assert cr.get_value("included", "value") == "included" From 8ac5a30519b6f4af85398b9b9d7064ff4d452da2 Mon Sep 17 00:00:00 2001 From: "GPT 5.6" Date: Wed, 15 Jul 2026 16:44:13 +0200 Subject: [PATCH 1365/1392] fix: prevent environment expansion in clone URLs Repo.clone_from() expanded environment-variable references in caller-supplied URLs before passing them to git clone. This could expose process environment values to an untrusted remote and made protocol validation apply to a different value than Git received. Polish clone URLs without variable or home expansion on native and Cygwin Git, then apply unsafe-protocol validation to that exact normalized value. Preserve the literal URL when normalizing the stored origin after a successful clone, while retaining the existing Git.polish_url() default for callers that intentionally normalize local paths. Add regression coverage for POSIX and Windows variable syntax, Cygwin conversion, stored origins, and post-normalization protocol validation. Git baseline: git clone passes URL arguments through literally; t/t5601-clone.sh covers the accepted URL forms without shell-style environment expansion. Security: GHSA-rwj8-pgh3-r573. Co-authored-by: Sebastian Thiel --- git/cmd.py | 19 ++++++++++++------- git/repo/base.py | 7 ++++--- git/util.py | 15 +++++++++------ test/test_clone.py | 41 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 65 insertions(+), 17 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 2a4f36024..150c9d16f 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -903,29 +903,34 @@ def is_cygwin(cls) -> bool: @overload @classmethod - def polish_url(cls, url: str, is_cygwin: Literal[False] = ...) -> str: ... + def polish_url(cls, url: str, is_cygwin: Literal[False] = ..., expand_vars: bool = ...) -> str: ... @overload @classmethod - def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> str: ... + def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None, expand_vars: bool = True) -> str: ... @classmethod - def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None) -> PathLike: + def polish_url(cls, url: str, is_cygwin: Union[None, bool] = None, expand_vars: bool = True) -> PathLike: """Remove any backslashes from URLs to be written in config files. Windows might create config files containing paths with backslashes, but git stops liking them as it will escape the backslashes. Hence we undo the escaping just to be sure. + + :param expand_vars: + Expand environment variables and an initial ``~``. Disable this for values + obtained from an untrusted source, such as remote URLs. """ if is_cygwin is None: is_cygwin = cls.is_cygwin() if is_cygwin: - url = cygpath(url) + url = cygpath(url, expand_vars=expand_vars) else: - url = os.path.expandvars(url) - if url.startswith("~"): - url = os.path.expanduser(url) + if expand_vars: + url = os.path.expandvars(url) + if url.startswith("~"): + url = os.path.expanduser(url) url = url.replace("\\\\", "\\").replace("\\", "/") return url diff --git a/git/repo/base.py b/git/repo/base.py index 913984975..e478396b2 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -1452,8 +1452,9 @@ def _clone( if multi_options: multi = shlex.split(" ".join(multi_options)) + clone_url = Git.polish_url(url, expand_vars=False) if not allow_unsafe_protocols: - Git.check_unsafe_protocols(url) + Git.check_unsafe_protocols(clone_url) if not allow_unsafe_options: Git.check_unsafe_options( options=Git._option_candidates([], kwargs), @@ -1465,7 +1466,7 @@ def _clone( proc = git.clone( multi, "--", - Git.polish_url(url), + clone_url, clone_path, with_extended_output=True, as_process=True, @@ -1505,7 +1506,7 @@ def _clone( # escape the backslashes. Hence we undo the escaping just to be sure. if repo.remotes: with repo.remotes[0].config_writer as writer: - writer.set_value("url", Git.polish_url(repo.remotes[0].url)) + writer.set_value("url", Git.polish_url(repo.remotes[0].url, expand_vars=False)) # END handle remote repo return repo diff --git a/git/util.py b/git/util.py index 712fabe85..11c1a0b15 100644 --- a/git/util.py +++ b/git/util.py @@ -382,13 +382,13 @@ def is_exec(fpath: str) -> bool: return progs -def _cygexpath(drive: Optional[str], path: str) -> str: +def _cygexpath(drive: Optional[str], path: str, expand_vars: bool = True) -> str: if osp.isabs(path) and not drive: # Invoked from `cygpath()` directly with `D:Apps\123`? # It's an error, leave it alone just slashes) p = path # convert to str if AnyPath given else: - p = path and osp.normpath(osp.expandvars(osp.expanduser(path))) + p = path and osp.normpath(osp.expandvars(osp.expanduser(path)) if expand_vars else path) if osp.isabs(p): if drive: # Confusing, maybe a remote system should expand vars. @@ -416,7 +416,7 @@ def _cygexpath(drive: Optional[str], path: str) -> str: ) -def cygpath(path: str) -> str: +def cygpath(path: str, expand_vars: bool = True) -> str: """Use :meth:`git.cmd.Git.polish_url` instead, that works on any environment.""" path = os.fspath(path) # Ensure is str and not AnyPath. # Fix to use Paths when 3.5 dropped. Or to be just str if only for URLs? @@ -424,12 +424,15 @@ def cygpath(path: str) -> str: for regex, parser, recurse in _cygpath_parsers: match = regex.match(path) if match: - path = parser(*match.groups()) + if parser is _cygexpath: + path = parser(*match.groups(), expand_vars=expand_vars) + else: + path = parser(*match.groups()) if recurse: - path = cygpath(path) + path = cygpath(path, expand_vars=expand_vars) break else: - path = _cygexpath(None, path) + path = _cygexpath(None, path, expand_vars=expand_vars) return path diff --git a/test/test_clone.py b/test/test_clone.py index 79d63dfdc..5fd59b3a3 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -7,8 +7,9 @@ import sys import tempfile from unittest import skip +from unittest import mock -from git import GitCommandError, Repo +from git import Git, GitCommandError, Repo from git.exc import UnsafeOptionError, UnsafeProtocolError from test.lib import TestBase, with_rw_directory, with_rw_repo, PathLikeMock @@ -346,6 +347,44 @@ def test_clone_from_unsafe_protocol(self): Repo.clone_from(url, tmp_dir / "repo") assert not tmp_file.exists() + def test_clone_from_does_not_expand_environment_variables_in_url(self): + urls = [ + "https://example.com/$GITPYTHON_TEST_SECRET/repo.git", + "https://example.com/${GITPYTHON_TEST_SECRET}/repo.git", + "https://example.com/%GITPYTHON_TEST_SECRET%/repo.git", + ] + with mock.patch.dict(os.environ, {"GITPYTHON_TEST_SECRET": "sensitive-value"}): + for url in urls: + with mock.patch.object(Git, "_call_process", side_effect=RuntimeError) as call_process: + with self.assertRaises(RuntimeError): + Repo.clone_from(url, "unused") + + assert call_process.call_args[0][3] == url + + @with_rw_directory + def test_clone_from_does_not_expand_environment_variables_in_stored_url(self, rw_dir): + url = pathlib.Path(rw_dir) / "$GITPYTHON_TEST_SECRET" / "source" + Git().init(url) + + with mock.patch.dict(os.environ, {"GITPYTHON_TEST_SECRET": "sensitive-value"}): + cloned = Repo.clone_from(url, pathlib.Path(rw_dir) / "clone") + + assert cloned.remotes.origin.url == Git.polish_url(str(url), expand_vars=False) + + def test_clone_from_checks_polished_url_for_unsafe_protocol(self): + with mock.patch.object(Git, "polish_url", return_value="ext::command"): + with mock.patch.object(Git, "_call_process") as call_process: + with self.assertRaises(UnsafeProtocolError): + Repo.clone_from("$GITPYTHON_TEST_URL", "unused") + + call_process.assert_not_called() + + def test_polish_url_does_not_expand_environment_variables_for_cygwin(self): + urls = ["$GITPYTHON_TEST_SECRET/repo", "user@example.com:$GITPYTHON_TEST_SECRET/repo"] + with mock.patch.dict(os.environ, {"GITPYTHON_TEST_SECRET": "sensitive-value"}): + for url in urls: + assert Git.polish_url(url, is_cygwin=True, expand_vars=False) == url + def test_clone_from_unsafe_protocol_allowed(self): with tempfile.TemporaryDirectory() as tdir: tmp_dir = pathlib.Path(tdir) From f8b6df5f10033508b5f5dd26f3ab4a87b215c3ca Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 16 Jul 2026 05:13:53 +0200 Subject: [PATCH 1366/1392] bump patch level prior to release --- VERSION | 2 +- doc/source/changes.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9e5b44197..2c5a9fd4a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.51 +3.1.52 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 8337478dd..2bfe6dd17 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,15 @@ Changelog ========= +3.1.52 +====== + +A security fix for +https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-rwj8-pgh3-r573. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.52 + 3.1.51 ====== From d702d9a6f598bc5eb701b468940db56afb978059 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Mon, 20 Apr 2026 06:11:16 -0700 Subject: [PATCH 1367/1392] feat(submodule): add deinit method to Submodule (#2014) Mirrors the pattern of `Submodule.add`, `Submodule.update`, and `Submodule.remove` by exposing `git submodule deinit` as a first-class method, so callers no longer need the `repo.git.submodule('deinit', ...)` workaround noted in the issue. The method delegates to `git submodule deinit [--force] -- ` via `self.repo.git.submodule`, decorated with `@unbare_repo` to match the other mutating Submodule methods. It unregisters the submodule from `.git/config` and clears the working-tree directory while leaving `.gitmodules` and `.git/modules/` intact, so a later `update()` can re-initialize. --- git/objects/submodule/base.py | 31 +++++++++++++++++++++++++++++++ test/test_submodule.py | 15 +++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index d183672db..33b1a29a0 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -49,6 +49,7 @@ Callable, Dict, Iterator, + List, Mapping, Sequence, TYPE_CHECKING, @@ -1267,6 +1268,36 @@ def remove( return self + @unbare_repo + def deinit(self, force: bool = False) -> "Submodule": + """Run ``git submodule deinit`` on this submodule. + + This is a thin wrapper around ``git submodule deinit ``, paralleling + :meth:`add`, :meth:`update`, and :meth:`remove`. It unregisters the + submodule (removes its entry from ``.git/config`` and empties the + working-tree directory) without deleting the submodule from + ``.gitmodules`` or its checked-out repository under ``.git/modules/``. + A subsequent :meth:`update` will re-initialize the submodule from the + retained contents. + + :param force: + If ``True``, pass ``--force`` to ``git submodule deinit``. This + allows deinitialization even when the submodule's working tree has + local modifications that would otherwise block the command. + + :return: + self + + :note: + Doesn't work in bare repositories. + """ + args: List[str] = [] + if force: + args.append("--force") + args.extend(["--", str(self.path)]) + self.repo.git.submodule("deinit", *args) + return self + def set_parent_commit(self, commit: Union[Commit_ish, str, None], check: bool = True) -> "Submodule": """Set this instance to use the given commit whose tree is supposed to contain the ``.gitmodules`` blob. diff --git a/test/test_submodule.py b/test/test_submodule.py index 47647f2a1..a3705f452 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -718,6 +718,21 @@ def test_iter_items_from_invalid_hash(self): next(it) self.assertIsNone(ctx.exception.value) + @with_rw_directory + def test_deinit_calls_git_submodule(self, rwdir): + repo = git.Repo.init(rwdir) + submodule = Submodule(repo, b"\0" * 20, name="module", path="module") + + with mock.patch.object(Git, "submodule", create=True) as git_submodule: + submodule.deinit() + + git_submodule.assert_called_once_with("deinit", "--", submodule.path) + git_submodule.reset_mock() + + submodule.deinit(force=True) + + git_submodule.assert_called_once_with("deinit", "--force", "--", submodule.path) + @with_rw_repo(k_no_subm_tag, bare=False) def test_first_submodule(self, rwrepo): assert len(list(rwrepo.iter_submodules())) == 0 From 0c0cc8acab16733ec294af115aa909c6d8d9738f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 16 Jul 2026 06:44:05 +0200 Subject: [PATCH 1368/1392] review --- git/objects/submodule/base.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 33b1a29a0..9899b1eac 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1272,13 +1272,13 @@ def remove( def deinit(self, force: bool = False) -> "Submodule": """Run ``git submodule deinit`` on this submodule. - This is a thin wrapper around ``git submodule deinit ``, paralleling - :meth:`add`, :meth:`update`, and :meth:`remove`. It unregisters the - submodule (removes its entry from ``.git/config`` and empties the - working-tree directory) without deleting the submodule from - ``.gitmodules`` or its checked-out repository under ``.git/modules/``. - A subsequent :meth:`update` will re-initialize the submodule from the - retained contents. + This is a thin wrapper around ``git submodule deinit ``, + which unregisters the submodule (removes its entry from + ``.git/config`` and empties the working-tree directory) + without deleting the submodule from ``.gitmodules`` + or its checked-out repository under ``.git/modules/``. + A subsequent :meth:`update` will re-initialize the + submodule from the retained contents. :param force: If ``True``, pass ``--force`` to ``git submodule deinit``. This From 6d0b6a4609b15bb74d45abec2e3b79bdb4632634 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 16 Jul 2026 05:09:09 +0200 Subject: [PATCH 1369/1392] typing: introduce sensible basedpyright defaults That way it dosesn't all light up immediately in dev mode. --- git/cmd.py | 2 +- git/util.py | 46 ++++++++++++++++++++++++++++------------------ pyproject.toml | 11 +++++++++++ 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index 150c9d16f..1868862d3 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1358,7 +1358,7 @@ def execute( # Allow the user to have the command executed in their working dir. try: - cwd = self._working_dir or os.getcwd() # type: Union[None, str] + cwd = self._working_dir or os.getcwd() # type: Optional[PathLike] if not os.access(str(cwd), os.X_OK): cwd = None except FileNotFoundError: diff --git a/git/util.py b/git/util.py index 11c1a0b15..8584b0e7f 100644 --- a/git/util.py +++ b/git/util.py @@ -217,7 +217,7 @@ def rmtree(path: PathLike) -> None: couldn't be deleted are read-only. Windows will not remove them in that case. """ - def handler(function: Callable, path: PathLike, _excinfo: Any) -> None: + def handler(function: Callable[[str], Any], path: str, _excinfo: Any) -> None: """Callback for :func:`shutil.rmtree`. This works as either a ``onexc`` or ``onerror`` style callback. @@ -401,7 +401,7 @@ def _cygexpath(drive: Optional[str], path: str, expand_vars: bool = True) -> str return p_str.replace("\\", "/") -_cygpath_parsers: Tuple[Tuple[Pattern[str], Callable, bool], ...] = ( +_cygpath_parsers: Tuple[Tuple[Pattern[str], Callable[..., str], bool], ...] = ( # See: https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx # and: https://www.cygwin.com/cygwin-ug-net/using.html#unc-paths ( @@ -508,7 +508,7 @@ def get_user_id() -> str: return "%s@%s" % (getpass.getuser(), platform.node()) -def finalize_process(proc: Union[subprocess.Popen, "Git.AutoInterrupt"], **kwargs: Any) -> None: +def finalize_process(proc: Union["subprocess.Popen[Any]", "Git.AutoInterrupt"], **kwargs: Any) -> None: """Wait for the process (clone, fetch, pull or push) and handle its errors accordingly.""" # TODO: No close proc-streams?? @@ -520,19 +520,21 @@ def expand_path(p: None, expand_vars: bool = ...) -> None: ... @overload -def expand_path(p: PathLike, expand_vars: bool = ...) -> str: +def expand_path(p: PathLike, expand_vars: bool = ...) -> Optional[PathLike]: # TODO: Support for Python 3.5 has been dropped, so these overloads can be improved. ... def expand_path(p: Union[None, PathLike], expand_vars: bool = True) -> Optional[PathLike]: - if isinstance(p, Path): - return p.resolve() + if p is None: + return None try: - p = osp.expanduser(p) # type: ignore[arg-type] + if isinstance(p, Path): + return p.resolve() + expanded_path = osp.expanduser(os.fspath(p)) if expand_vars: - p = osp.expandvars(p) - return osp.normpath(osp.abspath(p)) + expanded_path = osp.expandvars(expanded_path) + return osp.normpath(osp.abspath(expanded_path)) except Exception: return None @@ -767,7 +769,7 @@ class CallableRemoteProgress(RemoteProgress): __slots__ = ("_callable",) - def __init__(self, fn: Callable) -> None: + def __init__(self, fn: Callable[..., Any]) -> None: self._callable = fn super().__init__() @@ -846,7 +848,7 @@ def _main_actor( cls, env_name: str, env_email: str, - config_reader: Union[None, "GitConfigParser", "SectionConstraint"] = None, + config_reader: Union[None, "GitConfigParser", "SectionConstraint[GitConfigParser]"] = None, ) -> "Actor": actor = Actor("", "") user_id = None # We use this to avoid multiple calls to getpass.getuser(). @@ -882,7 +884,9 @@ def default_name() -> str: return actor @classmethod - def committer(cls, config_reader: Union[None, "GitConfigParser", "SectionConstraint"] = None) -> "Actor": + def committer( + cls, config_reader: Union[None, "GitConfigParser", "SectionConstraint[GitConfigParser]"] = None + ) -> "Actor": """ :return: :class:`Actor` instance corresponding to the configured committer. It @@ -897,7 +901,9 @@ def committer(cls, config_reader: Union[None, "GitConfigParser", "SectionConstra return cls._main_actor(cls.env_committer_name, cls.env_committer_email, config_reader) @classmethod - def author(cls, config_reader: Union[None, "GitConfigParser", "SectionConstraint"] = None) -> "Actor": + def author( + cls, config_reader: Union[None, "GitConfigParser", "SectionConstraint[GitConfigParser]"] = None + ) -> "Actor": """Same as :meth:`committer`, but defines the main author. It may be specified in the environment, but defaults to the committer.""" return cls._main_actor(cls.env_author_name, cls.env_author_email, config_reader) @@ -980,11 +986,11 @@ class IndexFileSHA1Writer: __slots__ = ("f", "sha1") - def __init__(self, f: IO) -> None: + def __init__(self, f: IO[bytes]) -> None: self.f = f self.sha1 = make_sha(b"") - def write(self, data: AnyStr) -> int: + def write(self, data: bytes) -> int: self.sha1.update(data) return self.f.write(data) @@ -1181,6 +1187,7 @@ def __new__(cls, id_attr: str, prefix: str = "") -> "IterableList[T_IterableObj] return super().__new__(cls) def __init__(self, id_attr: str, prefix: str = "") -> None: + super().__init__() self._id_attr = id_attr self._prefix = prefix @@ -1210,7 +1217,9 @@ def __getattr__(self, attr: str) -> T_IterableObj: # END for each item return list.__getattribute__(self, attr) - def __getitem__(self, index: Union[SupportsIndex, int, slice, str]) -> T_IterableObj: # type: ignore[override] + def __getitem__( # type: ignore[override] # pyright: ignore[reportIncompatibleMethodOverride] + self, index: Union[SupportsIndex, int, slice, str] + ) -> T_IterableObj: if isinstance(index, int): return list.__getitem__(self, index) elif isinstance(index, slice): @@ -1288,7 +1297,7 @@ def list_items(cls, repo: "Repo", *args: Any, **kwargs: Any) -> IterableList[T_I :return: list(Item,...) list of item instances """ - out_list: IterableList = IterableList(cls._id_attribute_) + out_list: IterableList[T_IterableObj] = IterableList(cls._id_attribute_) out_list.extend(cls.iter_items(repo, *args, **kwargs)) return out_list @@ -1297,7 +1306,8 @@ class IterableClassWatcher(type): """Metaclass that issues :exc:`DeprecationWarning` when :class:`git.util.Iterable` is subclassed.""" - def __init__(cls, name: str, bases: Tuple, clsdict: Dict) -> None: + def __init__(cls, name: str, bases: Tuple[type, ...], clsdict: Dict[str, Any]) -> None: + super().__init__(name, bases, clsdict) for base in bases: if type(base) is IterableClassWatcher: warnings.warn( diff --git a/pyproject.toml b/pyproject.toml index 149f2dc92..324630002 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,17 @@ exclude = ["^git/ext/gitdb"] module = "gitdb.*" ignore_missing_imports = true +[tool.basedpyright] +typeCheckingMode = "standard" +pythonVersion = "3.7" +extraPaths = [ + "git/ext/gitdb", + "git/ext/gitdb/gitdb/ext/smmap", +] +exclude = [ + "git/ext/gitdb", +] + [tool.coverage.run] source = ["git"] From b9d82c08210d107af9e2bdf7e368fb2f9dfec296 Mon Sep 17 00:00:00 2001 From: "GPT 5.6" Date: Thu, 16 Jul 2026 07:19:56 +0200 Subject: [PATCH 1370/1392] fix: make `submodule.update()` after `submodule.deinit()` work Git's `submodule deinit` command removes the submodule checkout but retains its repository under the parent repository's `.git/modules` directory. Submodule.update() previously treated the missing checkout as a completely uninitialized submodule and attempted to clone it again. The clone could not reuse the existing module repository, preventing a deinitialized submodule from being initialized again through GitPython. Detect a valid retained repository before entering the clone path. Restore the checkout's `.git` file and the module repository's worktree configuration, reset the retained repository to recreate its index and working tree, and restore the submodule URL in the parent configuration. Continue using the existing clone behavior when no valid retained repository is available. Co-authored-by: Sebastian Thiel --- git/objects/submodule/base.py | 210 ++++++++++++++++++---------------- test/test_submodule.py | 145 +++++++++++++++++++++++ 2 files changed, 255 insertions(+), 100 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 9899b1eac..7984db340 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -739,122 +739,132 @@ def update( mrepo = None # END init mrepo + def fetch_remotes(module_repo: "Repo") -> None: + rmts = module_repo.remotes + len_rmts = len(rmts) + for i, remote in enumerate(rmts): + op = FETCH + if i == 0: + op |= BEGIN + # END handle start + + progress.update( + op, + i, + len_rmts, + prefix + "Fetching remote %s of submodule %r" % (remote, self.name), + ) + # =============================== + if not dry_run: + remote.fetch(progress=progress) + # END handle dry-run + # =============================== + if i == len_rmts - 1: + op |= END + # END handle end + progress.update( + op, + i, + len_rmts, + prefix + "Done fetching remote of submodule %r" % self.name, + ) + # END fetch new data + try: # ENSURE REPO IS PRESENT AND UP-TO-DATE ####################################### try: mrepo = self.module() - rmts = mrepo.remotes - len_rmts = len(rmts) - for i, remote in enumerate(rmts): - op = FETCH - if i == 0: - op |= BEGIN - # END handle start - - progress.update( - op, - i, - len_rmts, - prefix + "Fetching remote %s of submodule %r" % (remote, self.name), - ) - # =============================== - if not dry_run: - remote.fetch(progress=progress) - # END handle dry-run - # =============================== - if i == len_rmts - 1: - op |= END - # END handle end - progress.update( - op, - i, - len_rmts, - prefix + "Done fetching remote of submodule %r" % self.name, - ) - # END fetch new data + fetch_remotes(mrepo) except InvalidGitRepositoryError: mrepo = None if not init: return self # END early abort if init is not allowed - # There is no git-repository yet - but delete empty paths. checkout_module_abspath = self.abspath - if not dry_run and osp.isdir(checkout_module_abspath): + module_abspath = self._module_abspath(self.repo, self.path, self.name) + + # ``git submodule deinit`` leaves the repository in + # ``.git/modules`` and empties the checkout. Reconnect that retained + # repository instead of trying to clone over it. + if not dry_run and osp.isdir(module_abspath): try: - os.rmdir(checkout_module_abspath) - except OSError as e: - raise OSError( - "Module directory at %r does already exist and is non-empty" % checkout_module_abspath - ) from e - # END handle OSError - # END handle directory removal - - # Don't check it out at first - nonetheless it will create a local - # branch according to the remote-HEAD if possible. - progress.update( - BEGIN | CLONE, - 0, - 1, - prefix - + "Cloning url '%s' to '%s' in submodule %r" % (self.url, checkout_module_abspath, self.name), - ) - if not dry_run: - if self.url.startswith("."): - url = urllib.parse.urljoin(self.repo.remotes.origin.url + "/", self.url) + git.Repo(module_abspath) + except InvalidGitRepositoryError: + pass else: - url = self.url - mrepo = self._clone_repo( - self.repo, - url, - self.path, - self.name, - n=True, - env=env, - multi_options=clone_multi_options, - allow_unsafe_options=allow_unsafe_options, - allow_unsafe_protocols=allow_unsafe_protocols, + if osp.lexists(checkout_module_abspath) and ( + osp.islink(checkout_module_abspath) + or not osp.isdir(checkout_module_abspath) + or os.listdir(checkout_module_abspath) + ): + raise OSError( + "Module directory at %r does already exist and is non-empty" % checkout_module_abspath + ) + os.makedirs(checkout_module_abspath, exist_ok=True) + self._write_git_file_and_module_config(checkout_module_abspath, module_abspath) + mrepo = git.Repo(checkout_module_abspath) + mrepo.head.reset(mrepo.head.commit, index=True, working_tree=True) + fetch_remotes(mrepo) + with self.repo.config_writer() as writer: + writer.set_value(sm_section(self.name), "url", self.url) + + if mrepo is None: + # There is no git-repository yet - but delete empty paths. + if not dry_run and osp.isdir(checkout_module_abspath): + try: + os.rmdir(checkout_module_abspath) + except OSError as e: + raise OSError( + "Module directory at %r does already exist and is non-empty" % checkout_module_abspath + ) from e + # END handle directory removal + + # Don't check it out at first - nonetheless it will create a local + # branch according to the remote-HEAD if possible. + progress.update( + BEGIN | CLONE, + 0, + 1, + prefix + + "Cloning url '%s' to '%s' in submodule %r" % (self.url, checkout_module_abspath, self.name), ) - # END handle dry-run - progress.update( - END | CLONE, - 0, - 1, - prefix + "Done cloning to %s" % checkout_module_abspath, - ) - - if not dry_run: - # See whether we have a valid branch to check out. - try: - mrepo = cast("Repo", mrepo) - # Find a remote which has our branch - we try to be flexible. - remote_branch = find_first_remote_branch(mrepo.remotes, self.branch_name) - local_branch = mkhead(mrepo, self.branch_path) - - # Have a valid branch, but no checkout - make sure we can figure - # that out by marking the commit with a null_sha. - local_branch.set_object(Object(mrepo, self.NULL_BIN_SHA)) - # END initial checkout + branch creation - - # Make sure HEAD is not detached. - mrepo.head.set_reference( - local_branch, - logmsg="submodule: attaching head to %s" % local_branch, + if not dry_run: + if self.url.startswith("."): + url = urllib.parse.urljoin(self.repo.remotes.origin.url + "/", self.url) + else: + url = self.url + mrepo = self._clone_repo( + self.repo, + url, + self.path, + self.name, + n=True, + env=env, + multi_options=clone_multi_options, + allow_unsafe_options=allow_unsafe_options, + allow_unsafe_protocols=allow_unsafe_protocols, ) - mrepo.head.reference.set_tracking_branch(remote_branch) - except (IndexError, InvalidGitRepositoryError): - _logger.warning("Failed to checkout tracking branch %s", self.branch_path) - # END handle tracking branch - - # NOTE: Have to write the repo config file as well, otherwise the - # default implementation will be offended and not update the - # repository. Maybe this is a good way to ensure it doesn't get into - # our way, but we want to stay backwards compatible too... It's so - # redundant! - with self.repo.config_writer() as writer: - writer.set_value(sm_section(self.name), "url", self.url) - # END handle dry_run + progress.update(END | CLONE, 0, 1, prefix + "Done cloning to %s" % checkout_module_abspath) + + if not dry_run: + # See whether we have a valid branch to check out. + try: + mrepo = cast("Repo", mrepo) + remote_branch = find_first_remote_branch(mrepo.remotes, self.branch_name) + local_branch = mkhead(mrepo, self.branch_path) + local_branch.set_object(Object(mrepo, self.NULL_BIN_SHA)) + mrepo.head.set_reference( + local_branch, + logmsg="submodule: attaching head to %s" % local_branch, + ) + mrepo.head.reference.set_tracking_branch(remote_branch) + except (IndexError, InvalidGitRepositoryError): + _logger.warning("Failed to checkout tracking branch %s", self.branch_path) + + with self.repo.config_writer() as writer: + writer.set_value(sm_section(self.name), "url", self.url) # END handle initialization # DETERMINE SHAS TO CHECK OUT diff --git a/test/test_submodule.py b/test/test_submodule.py index 0cd7c8e50..265ee2ffb 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -728,6 +728,151 @@ def test_deinit_calls_git_submodule(self, rwdir): git_submodule.assert_called_once_with("deinit", "--force", "--", submodule.path) + @with_rw_directory + def test_update_after_deinit(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + touch(osp.join(source_path, "file")) + source_repo.index.add(["file"]) + source_repo.index.commit("initial commit") + + parent_path = osp.join(rwdir, "parent") + parent_repo = git.Repo.init(parent_path) + submodule = parent_repo.create_submodule("module", "module", source_path) + parent_repo.index.commit("add submodule") + + submodule.deinit() + assert not submodule.module_exists() + assert osp.isdir(osp.join(parent_repo.git_dir, "modules", submodule.name)) + + submodule.update() + + assert submodule.module_exists() + assert submodule.module().head.commit == source_repo.head.commit + assert osp.isfile(osp.join(submodule.abspath, "file")) + + @with_rw_directory + def test_update_to_latest_revision_after_deinit_fetches_remote(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + source_repo.git.commit(m="initial commit", allow_empty=True) + + parent_repo = git.Repo.init(osp.join(rwdir, "parent")) + submodule = parent_repo.create_submodule("module", "module", source_path) + parent_repo.index.commit("add submodule") + submodule.deinit() + + touch(osp.join(source_path, "new-file")) + source_repo.index.add(["new-file"]) + source_repo.index.commit("advance remote") + + submodule.update(to_latest_revision=True) + + assert submodule.module().head.commit == source_repo.head.commit + assert osp.isfile(osp.join(submodule.abspath, "new-file")) + + @with_rw_directory + def test_update_after_deinit_fetches_new_gitlink_commit(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + source_repo.git.commit(m="initial commit", allow_empty=True) + + parent_repo = git.Repo.init(osp.join(rwdir, "parent")) + submodule = parent_repo.create_submodule("module", "module", source_path) + parent_repo.index.commit("add submodule") + submodule.deinit() + + touch(osp.join(source_path, "new-file")) + source_repo.index.add(["new-file"]) + source_repo.index.commit("advance remote") + parent_repo.git.update_index( + "--cacheinfo", + f"160000,{source_repo.head.commit.hexsha},{submodule.path}", + ) + parent_repo.index.commit("advance submodule") + + submodule = parent_repo.submodule(submodule.name) + submodule.update() + + assert submodule.module().head.commit == source_repo.head.commit + assert osp.isfile(osp.join(submodule.abspath, "new-file")) + + @with_rw_directory + def test_update_after_deinit_refuses_non_empty_checkout(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + tracked_file = osp.join(source_path, "file") + with open(tracked_file, "w") as fp: + fp.write("submodule content") + source_repo.index.add(["file"]) + source_repo.index.commit("initial commit") + + parent_repo = git.Repo.init(osp.join(rwdir, "parent")) + submodule = parent_repo.create_submodule("module", "module", source_path) + parent_repo.index.commit("add submodule") + submodule.deinit() + + checkout_file = osp.join(submodule.abspath, "file") + with open(checkout_file, "w") as fp: + fp.write("user content") + + with pytest.raises(OSError, match="does already exist and is non-empty"): + submodule.update() + + with open(checkout_file) as fp: + assert fp.read() == "user content" + assert not osp.exists(osp.join(submodule.abspath, ".git")) + + @with_rw_directory + def test_update_after_deinit_without_init_does_not_restore_checkout(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + source_repo.git.commit(m="initial commit", allow_empty=True) + + parent_repo = git.Repo.init(osp.join(rwdir, "parent")) + submodule = parent_repo.create_submodule("module", "module", source_path) + parent_repo.index.commit("add submodule") + submodule.deinit() + + assert submodule.update(init=False) is submodule + assert not submodule.module_exists() + assert not osp.exists(osp.join(submodule.abspath, ".git")) + + @with_rw_directory + def test_dry_run_update_after_deinit_does_not_restore_checkout(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + source_repo.git.commit(m="initial commit", allow_empty=True) + + parent_repo = git.Repo.init(osp.join(rwdir, "parent")) + submodule = parent_repo.create_submodule("module", "module", source_path) + parent_repo.index.commit("add submodule") + submodule.deinit() + + assert submodule.update(dry_run=True) is submodule + assert not submodule.module_exists() + assert not osp.exists(osp.join(submodule.abspath, ".git")) + + @with_rw_directory + def test_update_after_deinit_restores_nested_checkout(self, rwdir): + source_path = osp.join(rwdir, "source") + source_repo = git.Repo.init(source_path) + touch(osp.join(source_path, "file")) + source_repo.index.add(["file"]) + source_repo.index.commit("initial commit") + + parent_repo = git.Repo.init(osp.join(rwdir, "parent")) + submodule = parent_repo.create_submodule("nested/module", "deps/module", source_path) + parent_repo.index.commit("add nested submodule") + submodule.deinit() + + submodule.update() + + module_repo = submodule.module() + assert module_repo.head.commit == source_repo.head.commit + assert osp.isfile(osp.join(submodule.abspath, "file")) + assert osp.samefile(module_repo.git_dir, osp.join(parent_repo.git_dir, "modules", submodule.name)) + @with_rw_repo(k_no_subm_tag, bare=False) def test_first_submodule(self, rwrepo): assert len(list(rwrepo.iter_submodules())) == 0 From 5bc256060cfaf8f32100780f17e0846f3e69d8b7 Mon Sep 17 00:00:00 2001 From: Byron Date: Mon, 20 Jul 2026 09:06:41 +0200 Subject: [PATCH 1371/1392] upate contributing guide to avoid agent impersonation --- CONTRIBUTING.md | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 60e34a651..76f276323 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,25 +20,16 @@ A contribution that works only narrowly but lowers the quality of the codebase may be declined. The maintainers may not always be able to provide detailed feedback. -## AI-assisted contributions +## Prevent agent impersonation -If AI edits files for you, disclose it in the pull request description and commit -metadata. Prefer making the agent identity part of the commit, for example by using -an AI author such as `$agent $version ` or a co-author via -a `Co-authored-by: ` trailer. +AI agents communicating through a person's account must identify themselves, for +example in issue or PR descriptions and comments. AI assistance that does not replace +the person as the speaker, such as proofreading or wording polish, does not require +identification. -Agents operating through a person's GitHub account must identify themselves. For -example, comments posted by an agent should say so directly with phrases like -`AI agent on behalf of : ...`. - -Fully AI-generated comments on pull requests or issues must also be disclosed. -Undisclosed AI-generated comments may lead to the pull request or issue being closed. - -AI-assisted proofreading or wording polish does not need disclosure, but it is still -courteous to mention it when the AI materially influenced the final text. - -Automated or "full-auto" AI contributions without a human responsible for reviewing -and standing behind the work may be closed. +Attributing AI assistance in commit metadata, for example with a `Co-authored-by` +trailer, is welcome but not required. Code is reviewed the same way regardless of its +origin. ## Fuzzing Test Specific Documentation From 99bb94ecdb6c0233de35a6094853b65b1c301620 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 12 Apr 2026 13:32:15 +0000 Subject: [PATCH 1372/1392] Bump Vampire/setup-wsl from 6.0.0 to 7.0.0 Bumps [Vampire/setup-wsl](https://github.com/vampire/setup-wsl) from 6.0.0 to 7.0.0. - [Release notes](https://github.com/vampire/setup-wsl/releases) - [Commits](https://github.com/vampire/setup-wsl/compare/v6.0.0...v7.0.0) --- updated-dependencies: - dependency-name: Vampire/setup-wsl dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index fe3e26236..553062f2e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -66,7 +66,7 @@ jobs: - name: Set up WSL (Windows) if: matrix.os-type == 'windows' - uses: Vampire/setup-wsl@v6.0.0 + uses: Vampire/setup-wsl@v7.0.0 with: wsl-version: 1 distribution: Debian From 406b98e1695dfaa706304103b0d524515ace2ebf Mon Sep 17 00:00:00 2001 From: Siesta Date: Sun, 31 May 2026 21:42:58 +0800 Subject: [PATCH 1373/1392] fix: respect core.hooksPath for commit hooks Use git rev-parse --git-path when resolving commit hook paths so GitPython follows Git's core.hooksPath configuration.\n\nAdd regression coverage for a pre-commit hook stored in a custom hooks path.\n\nFixes #2083 --- git/index/fun.py | 12 +++++++++++- test/test_index.py | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/git/index/fun.py b/git/index/fun.py index 629c19b1e..71dd96e7f 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -64,6 +64,16 @@ def hook_path(name: str, git_dir: PathLike) -> str: return osp.join(git_dir, "hooks", name) +def _commit_hook_path(name: str, index: "IndexFile") -> str: + """:return: path to the named commit hook, respecting Git's core.hooksPath.""" + hp = index.repo.git.rev_parse("--git-path", f"hooks/{name}") + if osp.isabs(hp): + return hp + + base_dir = index.repo.working_dir or index.repo.git_dir + return osp.abspath(osp.join(base_dir, hp)) + + def _has_file_extension(path: str) -> str: return osp.splitext(path)[1] @@ -82,7 +92,7 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: :raise git.exc.HookExecutionError: """ - hp = hook_path(name, index.repo.git_dir) + hp = _commit_hook_path(name, index) if not os.access(hp, os.X_OK): return diff --git a/test/test_index.py b/test/test_index.py index 3be750dbb..ea8aa81ca 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1160,6 +1160,32 @@ def test_pre_commit_hook_success(self, rw_repo): _make_hook(index.repo.git_dir, "pre-commit", "exit 0") index.commit("This should not fail") + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.Absent, + reason="Can't run a hook on Windows without bash.exe.", + raises=HookExecutionError, + ) + @pytest.mark.xfail( + type(_win_bash_status) is WinBashStatus.WslNoDistro, + reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", + raises=HookExecutionError, + ) + @with_rw_repo("HEAD", bare=True) + def test_pre_commit_hook_respects_core_hooks_path(self, rw_repo): + index = rw_repo.index + hooks_dir = Path(index.repo.git_dir, "custom-hooks") + hooks_dir.mkdir() + hp = hooks_dir / "pre-commit" + hp.write_text(HOOKS_SHEBANG + "echo 'ran custom hook' >custom-hook-output.txt", encoding="utf-8") + os.chmod(hp, 0o744) + + with index.repo.config_writer() as writer: + writer.set_value("core", "hooksPath", "custom-hooks") + + index.commit("This should run the custom hook") + output = Path(rw_repo.git_dir, "custom-hook-output.txt").read_text(encoding="utf-8") + self.assertEqual(output, "ran custom hook\n") + @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", From 920652a4bd031c1fa1969b30993746fd910112f7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:32:54 +0000 Subject: [PATCH 1374/1392] Bump actions/setup-python from 6 to 7 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6 to 7. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index ffc12ca3a..e58084970 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -29,7 +29,7 @@ jobs: with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} From 9bc287a2b1eb331b6051d2ba704a603c3e0ddc6f Mon Sep 17 00:00:00 2001 From: Codex GPT-5 Date: Mon, 20 Jul 2026 10:24:12 +0200 Subject: [PATCH 1375/1392] Address review feedback about hook resolution - Review feedback: resolve core.hooksPath only when configured, avoid an unconditional git rev-parse dependency, and cover relative paths in a non-bare repository. - Read the effective Git configuration directly so unconfigured repositories retain the legacy .git/hooks lookup and configured relative paths resolve from the directory where hooks execute. Exercise both guarantees in the hook tests. - Review feedback: core.hooksPath can point outside index.repo.working_dir, where Path.relative_to() raises on Windows and prevents a valid hook from running. - Build the Bash argument with os.path.relpath so hooks in parent or other absolute locations remain executable. Fall back to the absolute POSIX-form path when Windows cannot form a relative path across drives, and add a focused command-construction regression test. --- git/index/fun.py | 21 ++++++++++++++------- test/test_index.py | 38 ++++++++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/git/index/fun.py b/git/index/fun.py index 71dd96e7f..5d52486f9 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -66,12 +66,13 @@ def hook_path(name: str, git_dir: PathLike) -> str: def _commit_hook_path(name: str, index: "IndexFile") -> str: """:return: path to the named commit hook, respecting Git's core.hooksPath.""" - hp = index.repo.git.rev_parse("--git-path", f"hooks/{name}") - if osp.isabs(hp): - return hp + with index.repo.config_reader() as config: + hooks_dir = config.get("core", "hooksPath", fallback="") - base_dir = index.repo.working_dir or index.repo.git_dir - return osp.abspath(osp.join(base_dir, hp)) + if not hooks_dir: + return hook_path(name, index.repo.git_dir) + + return osp.abspath(osp.join(index.repo.working_dir, osp.expanduser(hooks_dir), name)) def _has_file_extension(path: str) -> str: @@ -104,8 +105,14 @@ def run_commit_hook(name: str, index: "IndexFile", *args: str) -> None: if sys.platform == "win32" and not _has_file_extension(hp): # Windows only uses extensions to determine how to open files # (doesn't understand shebangs). Try using bash to run the hook. - relative_hp = Path(hp).relative_to(index.repo.working_dir).as_posix() - cmd = ["bash.exe", relative_hp] + try: + bash_hp = osp.relpath(hp, index.repo.working_dir) + except ValueError: + # Different drives have no relative path on Windows. Git Bash accepts + # an absolute path in this form, although a relative path is preferable + # because it also works with the Windows Subsystem for Linux wrapper. + bash_hp = hp + cmd = ["bash.exe", Path(bash_hp).as_posix()] process = safer_popen( cmd + list(args), diff --git a/test/test_index.py b/test/test_index.py index ea8aa81ca..881dec19f 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1099,9 +1099,35 @@ class Mocked: def test_run_commit_hook(self, rw_repo): index = rw_repo.index _make_hook(index.repo.git_dir, "fake-hook", "echo 'ran fake hook' >output.txt") - run_commit_hook("fake-hook", index) - output = Path(rw_repo.git_dir, "output.txt").read_text(encoding="utf-8") - self.assertEqual(output, "ran fake hook\n") + output = Path(rw_repo.git_dir, "output.txt") + with mock.patch.object(Repo, "config_level", ("repository",)): + with mock.patch.object(Git, "execute", side_effect=AssertionError("hook lookup must not run git")): + run_commit_hook("fake-hook", index) + self.assertEqual(output.read_text(encoding="utf-8"), "ran fake hook\n") + + output.unlink() + with index.repo.config_writer() as writer: + writer.set_value("core", "hooksPath", "") + run_commit_hook("fake-hook", index) + + self.assertEqual(output.read_text(encoding="utf-8"), "ran fake hook\n") + + @with_rw_directory + def test_run_commit_hook_outside_worktree_on_windows(self, rw_dir): + root = Path(rw_dir).resolve() + repo = Repo.init(root / "repo") + hooks_dir = root / "hooks" + _make_hook(root, "fake-hook", "exit 0") + with repo.config_writer() as writer: + writer.set_value("core", "hooksPath", str(hooks_dir)) + + with mock.patch("git.index.fun.sys.platform", "win32"): + with mock.patch("git.index.fun.safer_popen") as popen, mock.patch("git.index.fun.handle_process_output"): + popen.return_value.returncode = 0 + run_commit_hook("fake-hook", repo.index) + + command = popen.call_args[0][0] + self.assertEqual(command, ["bash.exe", "../hooks/fake-hook"]) @ddt.data((False,), (True,)) @with_rw_directory @@ -1170,10 +1196,10 @@ def test_pre_commit_hook_success(self, rw_repo): reason="Currently uses the bash.exe of WSL, even with no WSL distro installed", raises=HookExecutionError, ) - @with_rw_repo("HEAD", bare=True) + @with_rw_repo("HEAD") def test_pre_commit_hook_respects_core_hooks_path(self, rw_repo): index = rw_repo.index - hooks_dir = Path(index.repo.git_dir, "custom-hooks") + hooks_dir = Path(index.repo.working_dir, "custom-hooks") hooks_dir.mkdir() hp = hooks_dir / "pre-commit" hp.write_text(HOOKS_SHEBANG + "echo 'ran custom hook' >custom-hook-output.txt", encoding="utf-8") @@ -1183,7 +1209,7 @@ def test_pre_commit_hook_respects_core_hooks_path(self, rw_repo): writer.set_value("core", "hooksPath", "custom-hooks") index.commit("This should run the custom hook") - output = Path(rw_repo.git_dir, "custom-hook-output.txt").read_text(encoding="utf-8") + output = Path(rw_repo.working_dir, "custom-hook-output.txt").read_text(encoding="utf-8") self.assertEqual(output, "ran custom hook\n") @pytest.mark.xfail( From 1ed1b924f4e2d2ee7bab296df77b978af21853f1 Mon Sep 17 00:00:00 2001 From: "GPT 5.6" Date: Mon, 20 Jul 2026 14:13:22 +0200 Subject: [PATCH 1376/1392] fix: validate config section delimiters GHSA-3rp5-jjmw-4wv2 identified that configuration section names could alter the structure of serialized config despite the existing control-character checks. Reject unquoted closing section delimiters across all writer entry points while preserving valid delimiters inside quoted subsections and the existing option-name behavior. Add regression coverage for unsafe plain and quoted-subsection-shaped names as well as a valid quoted subsection. Git baseline: a23bace963d508bd96983cc637131392d3face18. Git config parsing treats an unquoted closing bracket as the end of a basic section header, while its extended form permits brackets inside a quoted subsection and requires a final bracket after the closing quote. Co-authored-by: Sebastian Thiel --- git/config.py | 16 +++++++++++++++ test/test_config.py | 47 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/git/config.py b/git/config.py index 64f501424..821710ea3 100644 --- a/git/config.py +++ b/git/config.py @@ -897,6 +897,22 @@ def _value_to_string_safe(self, value: Union[str, bytes, int, float, bool]) -> s def _assure_config_name_safe(self, name: "cp._SectionName", label: str) -> None: if isinstance(name, str) and UNSAFE_CONFIG_CHARS_RE.search(name): raise ValueError("Git config %s names must not contain CR, LF, or NUL" % label) + if label == "section" and isinstance(name, str): + in_quotes = False + escaped = False + for index, char in enumerate(name): + if escaped: + escaped = False + elif in_quotes and char == "\\": + escaped = True + elif char == '"': + if not in_quotes and (index == 0 or name[index - 1] not in " \t"): + raise ValueError("Git config quoted subsection names must begin after whitespace") + in_quotes = not in_quotes + elif char == "]" and not in_quotes: + raise ValueError("Git config section names must not contain an unquoted closing bracket") + if in_quotes: + raise ValueError("Git config section names must not contain an unterminated quote") @needs_values @set_dirty_and_flush_changes diff --git a/test/test_config.py b/test/test_config.py index c4e39db48..361a51fa9 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -194,6 +194,53 @@ def test_set_value_rejects_unsafe_section_and_option_names(self, rw_dir): self.assertEqual(git_config.get_value("user", "name"), "safe") self.assertFalse(git_config.has_section("core")) + @with_rw_directory + def test_writer_rejects_unquoted_section_terminators(self, rw_dir): + config_path = osp.join(rw_dir, "config") + bad_sections = ( + "user] [other", + 'user"] [other"', + 'submodule "docs"] [other', + 'submodule "docs] [other', + 'submodule "docs] [other\\', + ) + safe_section = 'submodule "docs]archive"' + + with GitConfigParser(config_path, read_only=False) as git_config: + git_config.add_section("user") + for bad_section in bad_sections: + with pytest.raises(ValueError, match="section name"): + git_config.add_section(bad_section) + with pytest.raises(ValueError, match="section name"): + git_config.set(bad_section, "name", "value") + with pytest.raises(ValueError, match="section name"): + git_config.set_value(bad_section, "name", "value") + with pytest.raises(ValueError, match="section name"): + git_config.add_value(bad_section, "name", "value") + with pytest.raises(ValueError, match="section name"): + git_config.rename_section("user", bad_section) + + git_config.set_value("user", "name", "safe") + git_config.set_value(safe_section, "name", "safe") + self.assertEqual(git_config.get_value(safe_section, "name"), "safe") + + # A closing bracket inside a quoted subsection name is data, not a section terminator. + with open(config_path, "rb") as config_file: + self.assertIn( + b'[submodule "docs]archive"]\n', + config_file.read(), + "a closing bracket within a quoted subsection name should be preserved", + ) + + # Reparse the file to verify that rejected names did not inject an [other] section. + with GitConfigParser(config_path, read_only=True) as git_config: + self.assertEqual( + git_config.get_value("user", "name"), + "safe", + "rejected section names corrupted the existing section", + ) + self.assertFalse(git_config.has_section("other"), "an unsafe section name injected an [other] section") + @with_rw_directory def test_set_and_add_value_reject_unsafe_value_characters(self, rw_dir): config_path = osp.join(rw_dir, "config") From faf3c09038b03bc2bdd8545ef34bbf6d7f1cd11f Mon Sep 17 00:00:00 2001 From: Byron Date: Mon, 20 Jul 2026 15:39:33 +0200 Subject: [PATCH 1377/1392] prepare for security fix --- VERSION | 2 +- doc/source/changes.rst | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2c5a9fd4a..406837cb8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.52 +3.1.53 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 2bfe6dd17..138826a3c 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,18 @@ Changelog ========= +3.1.53 +====== + +A security fix for +https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-3rp5-jjmw-4wv2 + +If you can, also try and provide feedback on the upcoming v4 branch +https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.53 + 3.1.52 ====== From e8d0fbf774d1f6baa3b481adfe48bd262e43b453 Mon Sep 17 00:00:00 2001 From: "GPT 5.6" Date: Wed, 22 Jul 2026 04:39:00 +0200 Subject: [PATCH 1378/1392] fix: validate split short-option values Single-character keyword arguments are transformed into an option token and a separate value token. The unsafe-option candidate builder only checked the keyword name, allowing an option-like value to bypass guards shared by clone, remote, revision, blame, and archive operations. Include dash-prefixed values only when short options are actually split, including sequence values, while preserving bare values and the non-splitting compatibility path. Git baseline a23bace9 defines clone -n and --upload-pack as distinct options, matching the argv boundary this validation now preserves. Refs GHSA-r9mr-m37c-5fr3. Co-authored-by: Sebastian Thiel --- git/cmd.py | 7 +++++++ test/test_git.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/git/cmd.py b/git/cmd.py index 1868862d3..5cba11f97 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1039,11 +1039,18 @@ def _option_candidates(cls, args: Sequence[Any] = (), kwargs: Optional[Mapping[s option for option in cls._unpack_args([arg for arg in args if arg is not None]) if option.startswith("-") ] if kwargs: + split_single_char_options = kwargs.get("split_single_char_options", True) for key, value in kwargs.items(): values = value if isinstance(value, (list, tuple)) else (value,) if any(value is True or (value is not False and value is not None) for value in values): key = str(key) options.append(f"-{key}" if len(key) == 1 else f"--{dashify(key)}") + if len(key) == 1 and split_single_char_options: + options.extend( + str(value) + for value in values + if value is not True and value not in (False, None) and str(value).startswith("-") + ) return options AutoInterrupt: TypeAlias = _AutoInterrupt diff --git a/test/test_git.py b/test/test_git.py index e95992ba8..a6698a021 100644 --- a/test/test_git.py +++ b/test/test_git.py @@ -214,6 +214,23 @@ def test_option_candidates_ignore_untransformed_kwargs(self): self.assertEqual(options, ["--max-count"]) + def test_option_candidates_include_split_single_char_option_values(self): + cases = [ + ({"n": "--upload-pack=helper"}, ["-n", "--upload-pack=helper"], ["--upload-pack"]), + ({"g": ("safe", "--out=target")}, ["-g", "--out=target"], ["--output"]), + ] + + for kwargs, candidates, unsafe_options in cases: + self.assertEqual(Git._option_candidates(kwargs=kwargs), candidates) + with self.assertRaises(UnsafeOptionError): + Git.check_unsafe_options(options=candidates, unsafe_options=unsafe_options) + + self.assertEqual(self.git.transform_kwargs(n="--upload-pack=helper"), ["-n", "--upload-pack=helper"]) + + unsplit_kwargs = {"n": "--upload-pack=helper", "split_single_char_options": False} + self.assertEqual(self.git.transform_kwargs(**unsplit_kwargs), ["-n--upload-pack=helper"]) + self.assertEqual(Git._option_candidates(kwargs=unsplit_kwargs), ["-n"]) + _shell_cases = ( # value_in_call, value_from_class, expected_popen_arg (None, False, False), From ffcb5359e87619f4fe4a70a4aff5f08c5580ba97 Mon Sep 17 00:00:00 2001 From: "GPT 5.6" Date: Wed, 22 Jul 2026 04:43:26 +0200 Subject: [PATCH 1379/1392] fix: reject unsafe clone templates Treat git clone --template as unsafe because caller-controlled templates can install hooks that execute during clone. Add regression coverage for both direct option and keyword forms. References GHSA-6p8h-3wgx-97gf. Validated against Git baseline a23bace9. Co-authored-by: Sebastian Thiel --- git/repo/base.py | 5 +++++ test/test_clone.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/git/repo/base.py b/git/repo/base.py index e478396b2..ae99ed5f9 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -149,6 +149,8 @@ class Repo: # Can override configuration variables that execute arbitrary commands: "--config", "-c", + # Can install hooks that execute during clone: + "--template", ] """Options to :manpage:`git-clone(1)` that allow arbitrary commands to be executed. @@ -159,6 +161,9 @@ class Repo: The ``--config``/``-c`` option allows users to override configuration variables like ``protocol.allow`` and ``core.gitProxy`` to execute arbitrary commands: https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---configltkeygtltvaluegt + + The ``--template`` option can install hooks that execute during clone: + https://git-scm.com/docs/git-clone#Documentation/git-clone.txt---templatetemplate-directory """ unsafe_git_archive_options = [ diff --git a/test/test_clone.py b/test/test_clone.py index 5fd59b3a3..e13d92f5b 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -128,6 +128,7 @@ def test_clone_unsafe_options(self, rw_repo): "-c protocol.ext.allow=always", "-cprotocol.ext.allow=always", "-vcprotocol.ext.allow=always", + f"--template={tmp_dir}", ] for unsafe_option in unsafe_options: with self.assertRaises(UnsafeOptionError): @@ -142,6 +143,7 @@ def test_clone_unsafe_options(self, rw_repo): {"config": "protocol.ext.allow=always"}, {"conf": "protocol.ext.allow=always"}, {"c": "protocol.ext.allow=always"}, + {"template": tmp_dir}, ] for unsafe_option in unsafe_options: with self.assertRaises(UnsafeOptionError): From 1d51b891d7f236044a6aa17498ec682b63dad6e6 Mon Sep 17 00:00:00 2001 From: "GPT 5.6" Date: Wed, 22 Jul 2026 04:46:13 +0200 Subject: [PATCH 1380/1392] fix: guard diff output options Reject unsafe diff options before revision parsing or Git invocation so callers cannot write command output to arbitrary filesystem paths. Cover commit and index diffs, including option-like revisions, and preserve an explicit allow_unsafe_options escape hatch. References GHSA-fjr4-x663-mwxc. Validated against Git baseline a23bace9. Co-authored-by: Sebastian Thiel --- git/diff.py | 14 ++++++++++++-- git/index/base.py | 18 ++++++++++++++++-- test/test_diff.py | 20 ++++++++++++++++++++ 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/git/diff.py b/git/diff.py index 5af53e556..24f79681a 100644 --- a/git/diff.py +++ b/git/diff.py @@ -9,7 +9,7 @@ import re import warnings -from git.cmd import handle_process_output +from git.cmd import Git, handle_process_output from git.compat import defenc from git.objects.blob import Blob from git.objects.util import mode_str_to_int @@ -35,7 +35,6 @@ if TYPE_CHECKING: from subprocess import Popen - from git.cmd import Git from git.objects.base import IndexObject from git.objects.commit import Commit from git.objects.tree import Tree @@ -190,6 +189,7 @@ def diff( other: Union[DiffConstants, "Tree", "Commit", str, None] = INDEX, paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, + allow_unsafe_options: bool = False, **kwargs: Any, ) -> "DiffIndex[Diff]": """Create diffs between two items being trees, trees and index or an index and @@ -219,6 +219,10 @@ def diff( applied makes the self to other. Patches are somewhat costly as blobs have to be read and diffed. + :param allow_unsafe_options: + If ``True``, allow options such as ``--output`` that can write to arbitrary + filesystem paths. + :param kwargs: Additional arguments passed to :manpage:`git-diff(1)`, such as ``R=True`` to swap both sides of the diff. @@ -231,6 +235,12 @@ def diff( an instance of :class:`~git.objects.tree.Tree` or :class:`~git.objects.commit.Commit`, or a git command error will occur. """ + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([other], kwargs), + unsafe_options=self.repo.unsafe_git_revision_options, + ) + args: List[Union[PathLike, Diffable]] = [] args.append("--abbrev=40") # We need full shas. args.append("--full-index") # Get full index paths, not only filenames. diff --git a/git/index/base.py b/git/index/base.py index f03b452dc..8d7dbfc1f 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -23,6 +23,7 @@ from gitdb.db import MemoryDB from git.compat import defenc, force_bytes +from git.cmd import Git import git.diff as git_diff from git.exc import CheckoutError, GitCommandError, GitError, InvalidGitRepositoryError from git.objects import Blob, Commit, Object, Submodule, Tree @@ -1492,6 +1493,7 @@ def diff( ] = git_diff.INDEX, paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, + allow_unsafe_options: bool = False, **kwargs: Any, ) -> git_diff.DiffIndex[git_diff.Diff]: """Diff this index against the working copy or a :class:`~git.objects.tree.Tree` @@ -1504,6 +1506,12 @@ def diff( Will only work with indices that represent the default git index as they have not been initialized with a stream. """ + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([other], kwargs), + unsafe_options=self.repo.unsafe_git_revision_options, + ) + # Only run if we are the default repository index. if self._file_path != self._index_path(): raise AssertionError("Cannot call %r on indices that do not represent the default git index" % self.diff()) @@ -1560,7 +1568,13 @@ def diff( # Invert the existing R flag. cur_val = kwargs.get("R", False) kwargs["R"] = not cur_val - return other.diff(self.INDEX, paths, create_patch, **kwargs) + return other.diff( + self.INDEX, + paths, + create_patch, + allow_unsafe_options=allow_unsafe_options, + **kwargs, + ) # END diff against other item handling # If other is not None here, something is wrong. @@ -1568,4 +1582,4 @@ def diff( raise ValueError("other must be None, Diffable.INDEX, a Tree or Commit, was %r" % other) # Diff against working copy - can be handled by superclass natively. - return super().diff(other, paths, create_patch, **kwargs) + return super().diff(other, paths, create_patch, allow_unsafe_options=allow_unsafe_options, **kwargs) diff --git a/test/test_diff.py b/test/test_diff.py index 612fbd9e0..a395f4a44 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -14,6 +14,7 @@ from git import NULL_TREE, Diff, DiffIndex, Diffable, GitCommandError, Repo, Submodule from git.cmd import Git +from git.exc import UnsafeOptionError from test.lib import StringProcessAdapter, TestBase, fixture, with_rw_directory @@ -352,6 +353,25 @@ def test_diff_submodule(self): self.assertIsInstance(diff.a_blob.size, int) self.assertIsInstance(diff.b_blob.size, int) + def test_diff_rejects_unsafe_output_options(self): + commit = self.rorepo.head.commit + + calls = ( + lambda target: commit.diff(output=target), + lambda target: commit.diff(other=f"--output={target}"), + lambda target: self.rorepo.index.diff(NULL_TREE, output=target), + lambda target: self.rorepo.index.diff(f"--output={target}"), + ) + for index, call in enumerate(calls): + target = osp.join(self.repo_dir, f"diff-output-{index}") + with self.assertRaises(UnsafeOptionError): + call(target) + self.assertFalse(osp.exists(target)) + + allowed_target = osp.join(self.repo_dir, "allowed-diff-output") + commit.diff(output=allowed_target, allow_unsafe_options=True) + self.assertTrue(osp.isfile(allowed_target)) + def test_diff_interface(self): """Test a few variations of the main diff routine.""" assertion_map = {} From e59d9bab02b095a97e179f47019afee95f4e3c18 Mon Sep 17 00:00:00 2001 From: Byron Date: Wed, 22 Jul 2026 06:07:15 +0200 Subject: [PATCH 1381/1392] prepare next release --- VERSION | 2 +- doc/source/changes.rst | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 406837cb8..8fc4b1976 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.53 +3.1.54 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 138826a3c..84d7b34a7 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,21 @@ Changelog ========= +3.1.54 +====== + +A security fix for + +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-r9mr-m37c-5fr3 +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-6p8h-3wgx-97gf +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-fjr4-x663-mwxc#event-857190 + +If you can, also try and provide feedback on the upcoming v4 branch +https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.54 + 3.1.53 ====== From 863417457a0633db7ea5aed4fd01e0b291a41162 Mon Sep 17 00:00:00 2001 From: "GPT 5.6" Date: Wed, 22 Jul 2026 15:42:44 +0200 Subject: [PATCH 1382/1392] fix: prevent environment expansion in remote URLs Remote and submodule creation expanded environment-variable references in caller-supplied URLs before storing them in Git configuration. This could expose process environment values through a later network request or a tracked .gitmodules file. Preserve variables literally in the remaining untrusted URL callers while retaining Git.polish_url() expansion for intentional local-path normalization. Add regression tests covering remote configuration and submodule configuration. Git baseline: git remote add and git submodule add accept URL arguments literally; Git does not perform shell-style environment expansion on those arguments. Co-authored-by: Sebastian Thiel --- git/objects/submodule/base.py | 2 +- git/remote.py | 2 +- test/test_remote.py | 9 +++++++++ test/test_submodule.py | 14 ++++++++++++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index 7984db340..97fc9e111 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -608,7 +608,7 @@ def add( # END verify url ## See #525 for ensuring git URLs in config-files are valid under Windows. - url = Git.polish_url(url) + url = Git.polish_url(url, expand_vars=False) # It's important to add the URL to the parent config, to let `git submodule` know. # Otherwise there is a '-' character in front of the submodule listing: diff --git a/git/remote.py b/git/remote.py index 0c3dbfe15..e2d5cbc1d 100644 --- a/git/remote.py +++ b/git/remote.py @@ -808,7 +808,7 @@ def create(cls, repo: "Repo", name: str, url: str, allow_unsafe_protocols: bool """ scmd = "add" kwargs["insert_kwargs_after"] = scmd - url = Git.polish_url(url) + url = Git.polish_url(url, expand_vars=False) if not allow_unsafe_protocols: Git.check_unsafe_protocols(url) repo.git.remote(scmd, "--", name, url, **kwargs) diff --git a/test/test_remote.py b/test/test_remote.py index 64c41290e..505d283af 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -4,6 +4,7 @@ # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ import gc +import os import os.path as osp from pathlib import Path import random @@ -685,6 +686,14 @@ def test_multiple_urls(self, rw_repo): # Will raise fatal: Will not delete all non-push URLs. self.assertRaises(GitCommandError, remote.delete_url, test3) + @with_rw_repo("HEAD", bare=False) + def test_create_does_not_expand_environment_variables_in_url(self, rw_repo): + url = "https://example.com/${GITPYTHON_TEST_SECRET}/repo.git" + with mock.patch.dict(os.environ, {"GITPYTHON_TEST_SECRET": "sensitive-value"}): + remote = rw_repo.create_remote("untrusted", url) + + assert remote.url == url + def test_fetch_error(self): rem = self.rorepo.remote("origin") msg = ( diff --git a/test/test_submodule.py b/test/test_submodule.py index 265ee2ffb..9879759dd 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -1355,6 +1355,20 @@ def test_add_no_clone_multi_options_argument(self, rwdir): with self.assertRaises(cp.NoOptionError): sm_config.get_value("core", "eol") + @with_rw_directory + def test_add_does_not_expand_environment_variables_in_url(self, rwdir): + parent = git.Repo.init(osp.join(rwdir, "parent")) + url = osp.join(rwdir, "${GITPYTHON_TEST_SECRET}") + source = git.Repo.init(url) + Path(source.working_tree_dir, "file").write_text("content") + source.index.add(["file"]) + source.index.commit("initial") + + with mock.patch.dict(os.environ, {"GITPYTHON_TEST_SECRET": "sensitive-value"}): + sm = Submodule.add(parent, "new", "new", url) + + assert sm.url == Git.polish_url(url, expand_vars=False) + @with_rw_repo("HEAD") def test_submodule_add_unsafe_url(self, rw_repo): with tempfile.TemporaryDirectory() as tdir: From 681c82c9c296f934635c81fa8294d4b6b6791b7e Mon Sep 17 00:00:00 2001 From: Byron Date: Thu, 23 Jul 2026 04:51:07 +0200 Subject: [PATCH 1383/1392] prepare release --- VERSION | 2 +- doc/source/changes.rst | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 8fc4b1976..5b1840f1e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.54 +3.1.55 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 84d7b34a7..a27ff48a2 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,19 @@ Changelog ========= +3.1.55 +====== + +A security fix for + +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-94p4-4cq8-9g67 + +If you can, also try and provide feedback on the upcoming v4 branch +https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.55 + 3.1.54 ====== @@ -9,7 +22,7 @@ A security fix for * https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-r9mr-m37c-5fr3 * https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-6p8h-3wgx-97gf -* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-fjr4-x663-mwxc#event-857190 +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-fjr4-x663-mwxc If you can, also try and provide feedback on the upcoming v4 branch https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. From 904aa71c58e0b0fecd2bcd94195b137841ce74f7 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:53:07 +0300 Subject: [PATCH 1384/1392] Declare support for Python 3.13-3.14 --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index a7b1eab00..00981ef27 100755 --- a/setup.py +++ b/setup.py @@ -108,5 +108,7 @@ def _stamp_version(filename: str) -> None: "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ], ) From 7d5370dab874f99c3327e2324e2c4fa65da5507e Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:56:34 +0300 Subject: [PATCH 1385/1392] Add support for Python 3.15 --- .github/workflows/pythonpackage.yml | 14 ++++++++++++-- setup.py | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index fe3e26236..eeb75ba6b 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: os-type: [ubuntu, macos, windows] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t", "3.14", "3.14t"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t", "3.14", "3.14t", "3.15", "3.15t"] exclude: - os-type: macos python-version: "3.7" # Not available for the ARM-based macOS runners. @@ -25,14 +25,20 @@ jobs: python-version: "3.13t" - os-type: macos python-version: "3.14t" + - os-type: macos + python-version: "3.15t" - os-type: windows - python-version: "3.13" # FIXME: Fix and enable Python 3.13 and 3.14 on Windows (#1955). + python-version: "3.13" # FIXME: Fix and enable Python 3.13-3.15 on Windows (#1955). - os-type: windows python-version: "3.13t" - os-type: windows python-version: "3.14" - os-type: windows python-version: "3.14t" + - os-type: windows + python-version: "3.15" + - os-type: windows + python-version: "3.15t" include: - os-ver: latest - os-type: ubuntu @@ -44,6 +50,10 @@ jobs: - python-version: "3.8" build-docs: false - experimental: false + - python-version: "3.15" + experimental: true + - python-version: "3.15t" + experimental: true fail-fast: false diff --git a/setup.py b/setup.py index 00981ef27..3a62a91fd 100755 --- a/setup.py +++ b/setup.py @@ -110,5 +110,6 @@ def _stamp_version(filename: str) -> None: "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", ], ) From 8a921820bf0f8af423f7ca6d14b1e17aa0f9d156 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:17:39 +0300 Subject: [PATCH 1386/1392] Stop testing experimental 3.13t --- .github/workflows/pythonpackage.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index eeb75ba6b..0f10e1e35 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -17,20 +17,16 @@ jobs: strategy: matrix: os-type: [ubuntu, macos, windows] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t", "3.14", "3.14t", "3.15", "3.15t"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15", "3.15t"] exclude: - os-type: macos python-version: "3.7" # Not available for the ARM-based macOS runners. - - os-type: macos - python-version: "3.13t" - os-type: macos python-version: "3.14t" - os-type: macos python-version: "3.15t" - os-type: windows python-version: "3.13" # FIXME: Fix and enable Python 3.13-3.15 on Windows (#1955). - - os-type: windows - python-version: "3.13t" - os-type: windows python-version: "3.14" - os-type: windows From 38553b6fddc7f6a667cdb45a6762343a08fc72b2 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 25 Jul 2026 06:27:01 +0200 Subject: [PATCH 1387/1392] fix: reject unsafe output options in Commit.count Commit.count forwards keyword arguments to git rev-list but did not apply the unsafe-option validation used by the sibling revision APIs. Validate forwarded options with the existing Git.check_unsafe_options helper and retain the explicit allow_unsafe_options escape hatch. This covers GHSA-p538-c434-8v24 without adding another option parser. Git baseline: git.git a23bace963 defines --output as a shared diff option consumed by setup_revisions; t/t6000-rev-list-misc.sh exercises that option with rev-list. Co-authored-by: GPT 5.6 --- git/objects/commit.py | 15 ++++++++++++++- test/test_commit.py | 5 +++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/git/objects/commit.py b/git/objects/commit.py index 1d8e8f071..3e435453d 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -269,13 +269,21 @@ def summary(self) -> Union[str, bytes]: else: return self.message.split(b"\n", 1)[0] - def count(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) -> int: + def count( + self, + paths: Union[PathLike, Sequence[PathLike]] = "", + allow_unsafe_options: bool = False, + **kwargs: Any, + ) -> int: """Count the number of commits reachable from this commit. :param paths: An optional path or a list of paths restricting the return value to commits actually containing the paths. + :param allow_unsafe_options: + Allow unsafe options, like ``--output``. + :param kwargs: Additional options to be passed to :manpage:`git-rev-list(1)`. They must not alter the output style of the command, or parsing will yield incorrect @@ -284,6 +292,11 @@ def count(self, paths: Union[PathLike, Sequence[PathLike]] = "", **kwargs: Any) :return: An int defining the number of reachable commits """ + if not allow_unsafe_options: + Git.check_unsafe_options( + options=Git._option_candidates([], kwargs), unsafe_options=self.unsafe_git_rev_options + ) + # Yes, it makes a difference whether empty paths are given or not in our case as # the empty paths version will ignore merge commits for some reason. if paths: diff --git a/test/test_commit.py b/test/test_commit.py index 74b7078f5..cb0427740 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -301,6 +301,11 @@ def test_iter_items_rejects_unsafe_options(self): with self.assertRaises(UnsafeOptionError): list(Commit.iter_items(self.rorepo, "HEAD", output=marker)) + def test_count_rejects_unsafe_options(self): + with tempfile.NamedTemporaryFile() as marker: + with self.assertRaises(UnsafeOptionError): + self.rorepo.head.commit.count(output=marker.name) + def test_rev_list_bisect_all(self): """ 'git rev-list --bisect-all' returns additional information From e3221f1252346259513e14afeed32f0a203512a7 Mon Sep 17 00:00:00 2001 From: Byron Date: Sat, 25 Jul 2026 09:30:05 +0200 Subject: [PATCH 1388/1392] prepare next release --- VERSION | 2 +- doc/source/changes.rst | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5b1840f1e..7db15315f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.55 +3.1.56 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index a27ff48a2..2f0e2a0ca 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,19 @@ Changelog ========= +3.1.56 +====== + +A security fix for + +* https://github.com/gitpython-developers/GitPython/security/advisories/GHSA-p538-c434-8v24 + +If you can, also try and provide feedback on the upcoming v4 branch +https://github.com/gitpython-developers/GitPython/pull/2177 - patches welcome. + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.55 + 3.1.55 ====== From 8a08afab30fed422e303e21f490542974a79a39a Mon Sep 17 00:00:00 2001 From: "GPT 5.6" Date: Tue, 21 Jul 2026 07:11:25 +0200 Subject: [PATCH 1389/1392] Build gitdb and smmap from the GitPython monorepo GitPython previously depended on gitdb and smmap through nested git submodules. That made a complete checkout depend on extra repository state and forced related changes to be coordinated across three repositories. Keep both projects as independently buildable distributions under their own top-level directories, while updating GitPython packaging, CI, documentation, and test fixtures to consume the in-tree sources. Remove the obsolete gitlinks and imported repository automation so a normal clone contains all code needed to build and test the three packages. Co-authored-by: Sebastian Thiel --- .github/dependabot.yml | 5 -- .github/workflows/alpine-test.yml | 6 +- .github/workflows/codeql.yml | 4 + .github/workflows/cygwin-test.yml | 16 +--- .github/workflows/dependencies.yml | 42 +++++++++ .github/workflows/pythonpackage.yml | 10 +-- .gitmodules | 3 - .pre-commit-config.yaml | 2 + README.md | 8 +- doc/source/intro.rst | 5 -- git/ext/gitdb | 1 - git/objects/fun.py | 4 +- gitdb/.github/dependabot.yml | 11 --- gitdb/.github/workflows/pythonpackage.yml | 52 ----------- gitdb/.gitmodules | 3 - gitdb/Makefile | 7 +- gitdb/README.rst | 15 ++-- gitdb/build-release.sh | 26 ------ gitdb/gitdb/__init__.py | 2 +- gitdb/gitdb/ext/smmap | 1 - gitdb/setup.py | 3 +- init-tests-after-clone.sh | 3 - pyproject.toml | 14 +-- setup.py | 2 +- smmap/.github/dependabot.yml | 6 -- smmap/.github/workflows/pythonpackage.yml | 49 ----------- smmap/Makefile | 7 +- smmap/README.md | 9 +- smmap/build-release.sh | 26 ------ smmap/setup.py | 6 +- smmap/smmap/__init__.py | 2 +- test/lib/helper.py | 23 ++++- test/test_clone.py | 3 + test/test_docs.py | 13 ++- test/test_fixture_health.py | 101 +++++----------------- test/test_fun.py | 4 + test/test_installation.py | 5 +- test/test_repo.py | 14 ++- test/test_submodule.py | 20 ++--- 39 files changed, 159 insertions(+), 374 deletions(-) create mode 100644 .github/workflows/dependencies.yml delete mode 100644 .gitmodules delete mode 160000 git/ext/gitdb delete mode 100644 gitdb/.github/dependabot.yml delete mode 100644 gitdb/.github/workflows/pythonpackage.yml delete mode 100644 gitdb/.gitmodules delete mode 100755 gitdb/build-release.sh delete mode 160000 gitdb/gitdb/ext/smmap delete mode 100644 smmap/.github/dependabot.yml delete mode 100644 smmap/.github/workflows/pythonpackage.yml delete mode 100755 smmap/build-release.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 16d5f11bc..7f803ee43 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,11 +6,6 @@ updates: schedule: interval: weekly -- package-ecosystem: gitsubmodule - directory: "/" - schedule: - interval: weekly - - package-ecosystem: pre-commit directory: "/" schedule: diff --git a/.github/workflows/alpine-test.yml b/.github/workflows/alpine-test.yml index a5b63f423..b10336a15 100644 --- a/.github/workflows/alpine-test.yml +++ b/.github/workflows/alpine-test.yml @@ -63,17 +63,13 @@ jobs: - name: Install project and test dependencies run: | . .venv/bin/activate - pip install '.[test]' + pip install ./smmap ./gitdb '.[test]' - name: Show POSIX file ownership run: | ls -ld -- \ "$(pwd)" \ "$(pwd)/.git" \ - "$(pwd)/git/ext/gitdb" \ - "$(pwd)/git/ext/gitdb/.git" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ "${HOME:?HOME is not set}/.gitconfig" \ 2>&1 || true diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1ad1a612c..eeb68cd0c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -62,6 +62,10 @@ jobs: with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} + config: | + paths-ignore: + - gitdb/gitdb/test/** + - smmap/smmap/test/** # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 6f9f347a9..b8150e93d 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -57,8 +57,6 @@ jobs: run: | git config --global --add safe.directory "$(pwd)" git config --global --add safe.directory "$(pwd)/.git" - git config --global --add safe.directory "$(pwd)/git/ext/gitdb" - git config --global --add safe.directory "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" git config --global core.autocrlf false - name: Prepare this repo for tests @@ -84,7 +82,7 @@ jobs: - name: Install project and test dependencies run: | - pip install '.[test]' + pip install ./smmap ./gitdb '.[test]' - name: Show POSIX file ownership # Cygwin's `ls -ld` reports the NTFS Owner SID via Cygwin's SID-to-uid @@ -96,12 +94,6 @@ jobs: ls -ld -- \ "$(pwd)" \ "$(pwd)/.git" \ - "$(pwd)/git/ext/gitdb" \ - "$(pwd)/git/ext/gitdb/.git" \ - "$(pwd)/.git/modules/gitdb" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ - "$(pwd)/.git/modules/gitdb/modules/smmap" \ "${HOME:?HOME is not set}/.gitconfig" \ 2>&1 || true @@ -114,12 +106,6 @@ jobs: $paths = @( "$pwd", "$pwd\.git", - "$pwd\git\ext\gitdb", - "$pwd\git\ext\gitdb\.git", - "$pwd\.git\modules\gitdb", - "$pwd\git\ext\gitdb\gitdb\ext\smmap", - "$pwd\git\ext\gitdb\gitdb\ext\smmap\.git", - "$pwd\.git\modules\gitdb\modules\smmap", "$env:USERPROFILE\.gitconfig" ) foreach ($p in $paths) { diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml new file mode 100644 index 000000000..4803e733b --- /dev/null +++ b/.github/workflows/dependencies.yml @@ -0,0 +1,42 @@ +name: Dependency packages + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ${{ matrix.python-version == '3.7' && 'ubuntu-22.04' || 'ubuntu-latest' }} + strategy: + fail-fast: false + matrix: + project: [smmap, gitdb] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - name: Install test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest flake8 ./smmap + if test "${{ matrix.project }}" = gitdb; then python -m pip install ./gitdb; fi + - name: Lint + run: flake8 "${{ matrix.project }}/${{ matrix.project }}" --count --select=E9,F63,F7,F82 --show-source --statistics + - name: Test + run: | + if test "${{ matrix.project }}" = gitdb; then + GITDB_TEST_GIT_REPO_BASE="$(git rev-parse --git-common-dir)" pytest -o addopts= -v gitdb/gitdb/test + else + pytest -o addopts= -v smmap/smmap/test + fi diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index fe3e26236..a95f3ded6 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -89,7 +89,7 @@ jobs: - name: Install project and test dependencies run: | - pip install '.[test]' + pip install ./smmap ./gitdb '.[test]' - name: Show POSIX file ownership # Linux and macOS only. On Windows, Git Bash's `ls -ld` reports a @@ -101,10 +101,6 @@ jobs: ls -ld -- \ "$(pwd)" \ "$(pwd)/.git" \ - "$(pwd)/git/ext/gitdb" \ - "$(pwd)/git/ext/gitdb/.git" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap" \ - "$(pwd)/git/ext/gitdb/gitdb/ext/smmap/.git" \ "${HOME:?HOME is not set}/.gitconfig" \ 2>&1 || true @@ -118,10 +114,6 @@ jobs: $paths = @( "$pwd", "$pwd\.git", - "$pwd\git\ext\gitdb", - "$pwd\git\ext\gitdb\.git", - "$pwd\git\ext\gitdb\gitdb\ext\smmap", - "$pwd\git\ext\gitdb\gitdb\ext\smmap\.git", "$env:USERPROFILE\.gitconfig" ) foreach ($p in $paths) { diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 251eeeec4..000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "gitdb"] - url = https://github.com/gitpython-developers/gitdb.git - path = git/ext/gitdb diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a13e85d40..0ec820905 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,5 @@ +exclude: ^(?:gitdb|smmap)/ + repos: - repo: https://github.com/codespell-project/codespell rev: v2.4.2 diff --git a/README.md b/README.md index 412d38205..c5a069bfe 100644 --- a/README.md +++ b/README.md @@ -101,15 +101,13 @@ In the less common case that you do not want to install test dependencies, `pip #### With editable *dependencies* (not preferred, and rarely needed) -In rare cases, you may want to work on GitPython and one or both of its [gitdb](https://github.com/gitpython-developers/gitdb) and [smmap](https://github.com/gitpython-developers/smmap) dependencies at the same time, with changes in your local working copy of gitdb or smmap immediately reflected in the behavior of your local working copy of GitPython. This can be done by making editable installations of those dependencies in the same virtual environment where you install GitPython. - -If you want to do that *and* you want the versions in GitPython's git submodules to be used, then pass `-e git/ext/gitdb` and/or `-e git/ext/gitdb/gitdb/ext/smmap` to `pip install`. This can be done in any order, and in separate `pip install` commands or the same one, so long as `-e` appears before *each* path. For example, you can install GitPython, gitdb, and smmap editably in the currently active virtual environment this way: +GitPython, [gitdb](gitdb), and [smmap](smmap) live in this repository as independently packaged projects. To work on all three at once, install each one editably in the same virtual environment: ```sh -pip install -e ".[test]" -e git/ext/gitdb -e git/ext/gitdb/gitdb/ext/smmap +pip install -e smmap -e gitdb -e ".[test]" ``` -The submodules must have been cloned for that to work, but that will already be the case if you have run `./init-tests-after-clone.sh`. You can use `pip list` to check which packages are installed editably and which are installed normally. +You can use `pip list` to check which packages are installed editably and which are installed normally. To reiterate, this approach should only rarely be used. For most development it is preferable to allow the gitdb and smmap dependencices to be retrieved automatically from PyPI in their latest stable packaged versions. diff --git a/doc/source/intro.rst b/doc/source/intro.rst index d053bd117..e1075b1c9 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -97,11 +97,6 @@ and cloned using:: $ git clone https://github.com/gitpython-developers/GitPython git-python -Initialize all submodules to obtain the required dependencies with:: - - $ cd git-python - $ git submodule update --init --recursive - Finally verify the installation by running unit tests:: $ python -m unittest diff --git a/git/ext/gitdb b/git/ext/gitdb deleted file mode 160000 index 335c0f661..000000000 --- a/git/ext/gitdb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 335c0f66173eecdc7b2597c2b6c3d1fde795df30 diff --git a/git/objects/fun.py b/git/objects/fun.py index fe57da13a..ad5fbd59b 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -115,11 +115,11 @@ def tree_entries_from_data(data: bytes) -> List[EntryTup]: # Default encoding for strings in git is UTF-8. # Only use the respective unicode object if the byte stream was encoded. name_bytes = data[ns:i] - name = safe_decode(name_bytes) + name = safe_decode(bytes(name_bytes)) # Byte is NULL, get next 20. i += 1 - sha = data[i : i + 20] + sha = bytes(data[i : i + 20]) i = i + 20 out.append((sha, mode, name)) # END for each byte in data stream diff --git a/gitdb/.github/dependabot.yml b/gitdb/.github/dependabot.yml deleted file mode 100644 index 2fe73ca77..000000000 --- a/gitdb/.github/dependabot.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 -updates: -- package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - -- package-ecosystem: "gitsubmodule" - directory: "/" - schedule: - interval: "weekly" diff --git a/gitdb/.github/workflows/pythonpackage.yml b/gitdb/.github/workflows/pythonpackage.yml deleted file mode 100644 index e58084970..000000000 --- a/gitdb/.github/workflows/pythonpackage.yml +++ /dev/null @@ -1,52 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python package - -on: [push, pull_request, workflow_dispatch] - -permissions: - contents: read - -jobs: - build: - - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] - os: [ubuntu-latest] - experimental: [false] - include: - - python-version: "3.7" - os: ubuntu-22.04 - experimental: false - continue-on-error: ${{ matrix.experimental }} - - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v7 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: ${{ matrix.experimental }} - - name: Install project and dependencies - run: | - python -m pip install --upgrade pip - pip install . - - name: Lint with flake8 - run: | - pip install flake8 - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pip install pytest - ulimit -n 48 - ulimit -n - pytest -v diff --git a/gitdb/.gitmodules b/gitdb/.gitmodules deleted file mode 100644 index e73cdedab..000000000 --- a/gitdb/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "smmap"] - path = gitdb/ext/smmap - url = https://github.com/gitpython-developers/smmap.git diff --git a/gitdb/Makefile b/gitdb/Makefile index 20436bbc4..ad0912ffe 100644 --- a/gitdb/Makefile +++ b/gitdb/Makefile @@ -1,12 +1,7 @@ -.PHONY: all clean release force_release +.PHONY: all clean all: @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all clean: rm -rf build/ dist/ .eggs/ .tox/ - -force_release: clean - ./build-release.sh - twine upload dist/* - git push --tags origin master diff --git a/gitdb/README.rst b/gitdb/README.rst index 61ce28b1e..45344a078 100644 --- a/gitdb/README.rst +++ b/gitdb/README.rst @@ -38,14 +38,9 @@ REQUIREMENTS SOURCE ====== -The source is available in a git repository on GitHub: +The source is available in the GitPython repository on GitHub: -https://github.com/gitpython-developers/gitdb - -Once the clone is complete, please be sure to initialize the submodule using:: - - cd gitdb - git submodule update --init +https://github.com/gitpython-developers/GitPython/tree/main/gitdb Run the tests with:: @@ -54,8 +49,8 @@ Run the tests with:: DEVELOPMENT =========== -.. image:: https://github.com/gitpython-developers/gitdb/workflows/Python%20package/badge.svg - :target: https://github.com/gitpython-developers/gitdb/actions +.. image:: https://github.com/gitpython-developers/GitPython/actions/workflows/dependencies.yml/badge.svg + :target: https://github.com/gitpython-developers/GitPython/actions/workflows/dependencies.yml The library is considered mature, and not under active development. Its primary (known) use is in GitPython. @@ -66,7 +61,7 @@ INFRASTRUCTURE * https://github.com/gitpython-developers/GitPython/discussions * Issue Tracker - * https://github.com/gitpython-developers/gitdb/issues + * https://github.com/gitpython-developers/GitPython/issues LICENSE ======= diff --git a/gitdb/build-release.sh b/gitdb/build-release.sh deleted file mode 100755 index 5840e4472..000000000 --- a/gitdb/build-release.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# -# This script builds a release. If run in a venv, it auto-installs its tools. -# You may want to run "make release" instead of running this script directly. - -set -eEu - -function release_with() { - $1 -m build --sdist --wheel -} - -if test -n "${VIRTUAL_ENV:-}"; then - deps=(build twine) # Install twine along with build, as we need it later. - echo "Virtual environment detected. Adding packages: ${deps[*]}" - pip install --quiet --upgrade "${deps[@]}" - echo 'Starting the build.' - release_with python -else - function suggest_venv() { - venv_cmd='python -m venv env && source env/bin/activate' - printf "HELP: To avoid this error, use a virtual-env with '%s' instead.\n" "$venv_cmd" - } - trap suggest_venv ERR # This keeps the original exit (error) code. - echo 'Starting the build.' - release_with python3 # Outside a venv, use python3. -fi diff --git a/gitdb/gitdb/__init__.py b/gitdb/gitdb/__init__.py index 1fb7df893..04920428d 100644 --- a/gitdb/gitdb/__init__.py +++ b/gitdb/gitdb/__init__.py @@ -6,7 +6,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" -__homepage__ = "https://github.com/gitpython-developers/gitdb" +__homepage__ = "https://github.com/gitpython-developers/GitPython/tree/main/gitdb" version_info = (4, 0, 12) __version__ = '.'.join(str(i) for i in version_info) diff --git a/gitdb/gitdb/ext/smmap b/gitdb/gitdb/ext/smmap deleted file mode 160000 index e4ad410ac..000000000 --- a/gitdb/gitdb/ext/smmap +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e4ad410ac3baf2046bd4043394e7cbb119045cc1 diff --git a/gitdb/setup.py b/gitdb/setup.py index 3a9154311..924f026b8 100755 --- a/gitdb/setup.py +++ b/gitdb/setup.py @@ -6,7 +6,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" -__homepage__ = "https://github.com/gitpython-developers/gitdb" +__homepage__ = "https://github.com/gitpython-developers/GitPython/tree/main/gitdb" version_info = (4, 0, 12) __version__ = '.'.join(str(i) for i in version_info) @@ -28,7 +28,6 @@ "Development Status :: 5 - Production/Stable", "Environment :: Console", "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Operating System :: POSIX", "Operating System :: Microsoft :: Windows", diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index a88f983fc..db3540857 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -55,9 +55,6 @@ git reset --hard HEAD~1 # Point the master branch where we started, so we test the correct code. git reset --hard __testing_point__ -# The tests need submodules, including a submodule with a submodule. -git submodule update --init --recursive - # The tests need some version tags. Try to get them even in forks. This fetches # other objects too. So, locally, we always do it, for a consistent experience. if ! ci || no_version_tags; then diff --git a/pyproject.toml b/pyproject.toml index 324630002..00662dceb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,20 +28,22 @@ warn_unreachable = true implicit_reexport = true # strict = true # TODO: Remove when 'gitdb' is fully annotated. -exclude = ["^git/ext/gitdb"] +exclude = ["^(gitdb|smmap)/"] [[tool.mypy.overrides]] -module = "gitdb.*" +module = ["gitdb", "gitdb.*"] +follow_imports = "skip" ignore_missing_imports = true [tool.basedpyright] typeCheckingMode = "standard" pythonVersion = "3.7" extraPaths = [ - "git/ext/gitdb", - "git/ext/gitdb/gitdb/ext/smmap", + "gitdb", + "smmap", ] exclude = [ - "git/ext/gitdb", + "gitdb", + "smmap", ] [tool.coverage.run] @@ -57,6 +59,8 @@ line-length = 120 # Exclude a variety of commonly ignored directories. exclude = [ "git/ext/", + "gitdb/", + "smmap/", "build", "dist", ] diff --git a/setup.py b/setup.py index a7b1eab00..6f95fef18 100755 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ def _stamp_version(filename: str) -> None: author_email="byronimo@gmail.com, mtrier@gmail.com", license="BSD-3-Clause", url="https://github.com/gitpython-developers/GitPython", - packages=find_packages(exclude=["test", "test.*"]), + packages=find_packages(include=["git", "git.*"]), include_package_data=True, package_dir={"git": "git"}, python_requires=">=3.7", diff --git a/smmap/.github/dependabot.yml b/smmap/.github/dependabot.yml deleted file mode 100644 index 203f3c889..000000000 --- a/smmap/.github/dependabot.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: 2 -updates: -- package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" diff --git a/smmap/.github/workflows/pythonpackage.yml b/smmap/.github/workflows/pythonpackage.yml deleted file mode 100644 index cedd8d849..000000000 --- a/smmap/.github/workflows/pythonpackage.yml +++ /dev/null @@ -1,49 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python package - -on: [push, pull_request, workflow_dispatch] - -permissions: - contents: read - -jobs: - build: - strategy: - matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] - include: - - experimental: false - - os: ubuntu-latest - - python-version: "3.7" - os: ubuntu-22.04 - - runs-on: ${{ matrix.os }} - - continue-on-error: ${{ matrix.experimental }} - - steps: - - uses: actions/checkout@v7 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: ${{ matrix.experimental }} - - name: Install project - run: | - python -m pip install --upgrade pip - pip install . - - name: Lint with flake8 - run: | - pip install flake8 - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test - run: | - pip install pytest - ulimit -n 48 - ulimit -n - pytest -v diff --git a/smmap/Makefile b/smmap/Makefile index 20436bbc4..ad0912ffe 100644 --- a/smmap/Makefile +++ b/smmap/Makefile @@ -1,12 +1,7 @@ -.PHONY: all clean release force_release +.PHONY: all clean all: @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all clean: rm -rf build/ dist/ .eggs/ .tox/ - -force_release: clean - ./build-release.sh - twine upload dist/* - git push --tags origin master diff --git a/smmap/README.md b/smmap/README.md index e21f53453..748f0f0db 100644 --- a/smmap/README.md +++ b/smmap/README.md @@ -13,7 +13,7 @@ Although memory maps have many advantages, they represent a very limited system ## Overview -![Python package](https://github.com/gitpython-developers/smmap/workflows/Python%20package/badge.svg) +![Dependency packages](https://github.com/gitpython-developers/GitPython/actions/workflows/dependencies.yml/badge.svg) Smmap wraps an interface around mmap and tracks the mapped files as well as the amount of clients who use it. If the system runs out of resources, or if a memory limit is reached, it will automatically unload unused maps to allow continued operation. @@ -56,11 +56,11 @@ It is advised to have a look at the **Usage Guide** for a brief introduction on ## Homepage and Links -The project is home on github at https://github.com/gitpython-developers/smmap . +The project is maintained in the GitPython repository at https://github.com/gitpython-developers/GitPython/tree/main/smmap . The latest source can be cloned from github as well: -* git://github.com/gitpython-developers/smmap.git +* https://github.com/gitpython-developers/GitPython.git For support, please use the git-python mailing list: @@ -70,7 +70,7 @@ For support, please use the git-python mailing list: Issues can be filed on github: -* https://github.com/gitpython-developers/smmap/issues +* https://github.com/gitpython-developers/GitPython/issues A link to the pypi page related to this repository: @@ -80,4 +80,3 @@ A link to the pypi page related to this repository: ## License Information *smmap* is licensed under the New BSD License. - diff --git a/smmap/build-release.sh b/smmap/build-release.sh deleted file mode 100755 index 5840e4472..000000000 --- a/smmap/build-release.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# -# This script builds a release. If run in a venv, it auto-installs its tools. -# You may want to run "make release" instead of running this script directly. - -set -eEu - -function release_with() { - $1 -m build --sdist --wheel -} - -if test -n "${VIRTUAL_ENV:-}"; then - deps=(build twine) # Install twine along with build, as we need it later. - echo "Virtual environment detected. Adding packages: ${deps[*]}" - pip install --quiet --upgrade "${deps[@]}" - echo 'Starting the build.' - release_with python -else - function suggest_venv() { - venv_cmd='python -m venv env && source env/bin/activate' - printf "HELP: To avoid this error, use a virtual-env with '%s' instead.\n" "$venv_cmd" - } - trap suggest_venv ERR # This keeps the original exit (error) code. - echo 'Starting the build.' - release_with python3 # Outside a venv, use python3. -fi diff --git a/smmap/setup.py b/smmap/setup.py index 7deb1788e..585f0e76c 100755 --- a/smmap/setup.py +++ b/smmap/setup.py @@ -11,9 +11,10 @@ import smmap if os.path.exists("README.md"): - long_description = open('README.md', encoding="utf-8").read().replace('\r\n', '\n') + with open("README.md", encoding="utf-8") as readme: + long_description = readme.read().replace("\r\n", "\n") else: - long_description = "See https://github.com/gitpython-developers/smmap" + long_description = "See https://github.com/gitpython-developers/GitPython/tree/main/smmap" setup( name="smmap", @@ -31,7 +32,6 @@ "Development Status :: 5 - Production/Stable", "Environment :: Console", "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Operating System :: POSIX", "Operating System :: Microsoft :: Windows", diff --git a/smmap/smmap/__init__.py b/smmap/smmap/__init__.py index 623181034..79ec0674d 100644 --- a/smmap/smmap/__init__.py +++ b/smmap/smmap/__init__.py @@ -2,7 +2,7 @@ __author__ = "Sebastian Thiel" __contact__ = "byronimo@gmail.com" -__homepage__ = "https://github.com/gitpython-developers/smmap" +__homepage__ = "https://github.com/gitpython-developers/GitPython/tree/main/smmap" version_info = (5, 0, 3) __version__ = '.'.join(str(i) for i in version_info) diff --git a/test/lib/helper.py b/test/lib/helper.py index 1c110e103..deddced2b 100644 --- a/test/lib/helper.py +++ b/test/lib/helper.py @@ -382,7 +382,13 @@ def _small_repo_url(self): """:return: A path to a small, clonable repository""" from git.cmd import Git - return Git.polish_url(osp.join(self.rorepo.working_tree_dir, "git/ext/gitdb/gitdb/ext/smmap")) + return Git.polish_url(self._dependency_repo_dirs["smmap"]) + + def _gitdb_repo_url(self): + """:return: A local gitdb repository reconstructed from merged history""" + from git.cmd import Git + + return Git.polish_url(self._dependency_repo_dirs["gitdb"]) @classmethod def setUpClass(cls): @@ -394,11 +400,26 @@ def setUpClass(cls): gc.collect() cls.rorepo = Repo(GIT_REPO) + cls._dependency_repo_root = tempfile.mkdtemp(prefix="gitpython-dependency-repos-") + cls._dependency_repo_dirs = {} + for name, rev in { + "gitdb": "2da3232f9d58e7761e384ac6d32f7b1ed77a74a2", + "smmap": "8ce61ad5cc4016bffaf25080bc0d69b3acbe8555", + }.items(): + path = osp.join(cls._dependency_repo_root, name) + repo = cls.rorepo.clone(path, shared=True, no_checkout=True) + repo.create_head("master", repo.commit(rev), force=True).checkout() + if name == "smmap": + repo.create_tag("v0.8.1", ref="master~10", message="Test fixture tag", force=True) + repo.git.gc() + repo.close() + cls._dependency_repo_dirs[name] = path @classmethod def tearDownClass(cls): cls.rorepo.git.clear_cache() cls.rorepo.git = None + rmtree(cls._dependency_repo_root) def _make_file(self, rela_path, data, repo=None): """ diff --git a/test/test_clone.py b/test/test_clone.py index 5fd59b3a3..3c8ed0a24 100644 --- a/test/test_clone.py +++ b/test/test_clone.py @@ -87,6 +87,9 @@ def test_clone_from_with_path_contains_unicode(self): Repo.clone_from( url=self._small_repo_url(), to_path=path_with_unicode, + # Local clones hardlink the reconstructed smmap repository's + # read-only metadata, which Windows cannot remove at cleanup. + no_hardlinks=True, ) except UnicodeEncodeError: self.fail("Raised UnicodeEncodeError") diff --git a/test/test_docs.py b/test/test_docs.py index c3426a807..c3cfec3e0 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -8,7 +8,7 @@ import os.path from test.lib import TestBase -from test.lib.helper import with_rw_directory +from test.lib.helper import with_rw_directory, with_rw_repo class Tutorials(TestBase): @@ -475,13 +475,20 @@ def test_references_and_objects(self, rw_dir): repo.git.clear_cache() - def test_submodules(self): + @with_rw_repo("c15a6e1923a14bc760851913858a3942a4193cdb") + def test_submodules(self, repo): # [1-test_submodules] - repo = self.rorepo sms = repo.submodules assert len(sms) == 1 sm = sms[0] + with sm.config_writer() as writer: + writer.set_value("url", self._gitdb_repo_url()) + sm.update(recursive=False) + child = sm.children()[0] + with child.config_writer() as writer: + writer.set_value("url", self._small_repo_url()) + child.update() self.assertEqual(sm.name, "gitdb") # GitPython has gitdb as its one and only (direct) submodule... self.assertEqual(sm.children()[0].name, "smmap") # ...which has smmap as its one and only submodule. diff --git a/test/test_fixture_health.py b/test/test_fixture_health.py index b18d5e8f9..8301001c4 100644 --- a/test/test_fixture_health.py +++ b/test/test_fixture_health.py @@ -1,18 +1,7 @@ # This module is part of GitPython and is released under the # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ -"""Verify that fixture directories are usable by git. - -If a fixture directory is missing, isn't an initialized git repository, -or is rejected by git for "dubious ownership", dependent tests -elsewhere in the suite fail in opaque ways. The checks here name the -preconditions directly so a misconfigured environment is recognizable -from the test output rather than from a cascade of unrelated-seeming -failures. - -These tests do not exercise GitPython's production code. They verify -the conditions under which production code is exercised are valid. -""" +"""Verify that the source repository is usable by git.""" import subprocess from pathlib import Path @@ -21,32 +10,25 @@ REPO_ROOT = Path(__file__).resolve().parent.parent -# Directories git must trust for the test suite to operate normally. The -# current set is the GitPython working tree plus the working trees of its -# gitdb submodule and the smmap submodule nested inside gitdb. New entries -# should be added here whenever the test suite gains a dependency on git -# accepting another directory. -FIXTURE_DIRS = [ - pytest.param(REPO_ROOT, id="repo_root"), - pytest.param(REPO_ROOT / "git" / "ext" / "gitdb", id="gitdb"), - pytest.param( - REPO_ROOT / "git" / "ext" / "gitdb" / "gitdb" / "ext" / "smmap", - id="smmap", - ), -] +FIXTURE_DIRS = [pytest.param(REPO_ROOT, id="repo_root")] + -# Submodule working trees that must be present and initialized for the -# test suite to operate normally: gitdb at `git/ext/gitdb`, and smmap -# nested inside gitdb at `git/ext/gitdb/gitdb/ext/smmap`. The paths -# below are anchored at REPO_ROOT (the GitPython source tree), not at -# any rorepo redirection target. -SUBMODULE_DIRS = [ - pytest.param(REPO_ROOT / "git" / "ext" / "gitdb", id="gitdb"), - pytest.param( - REPO_ROOT / "git" / "ext" / "gitdb" / "gitdb" / "ext" / "smmap", - id="smmap", - ), -] +def test_source_tree_has_no_gitlinks() -> None: + """All formerly external dependencies are regular monorepo directories.""" + if not (REPO_ROOT / ".git").exists(): + pytest.skip(f"{REPO_ROOT} is not a git checkout") + try: + result = subprocess.run( + ["git", "ls-files", "--stage"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + except FileNotFoundError: + pytest.skip("git is not installed or not on PATH") + gitlinks = [line for line in result.stdout.splitlines() if line.startswith("160000 ")] + assert not gitlinks, "Unexpected submodules:\n" + "\n".join(gitlinks) @pytest.mark.parametrize("fixture_dir", FIXTURE_DIRS) @@ -59,14 +41,8 @@ def test_fixture_dir_is_trusted_by_git(fixture_dir: Path) -> None: running user and the CI workflow's ``safe.directory`` list is missing an entry that would override the check. """ - if not fixture_dir.exists(): - pytest.skip(f"{fixture_dir} not present (run `git submodule update --init --recursive` from the repo root)") if not (fixture_dir / ".git").exists(): - pytest.skip( - f"{fixture_dir} has no .git marker " - "(submodule not initialized; run " - "`git submodule update --init --recursive` from the repo root)" - ) + pytest.skip(f"{fixture_dir} is not a git checkout") try: result = subprocess.run( ["git", "-C", str(fixture_dir), "rev-parse", "--show-toplevel"], @@ -92,40 +68,3 @@ def test_fixture_dir_is_trusted_by_git(fixture_dir: Path) -> None: "This usually means the directory is not an initialized git " "repository (its `.git` marker may be stale or pointing elsewhere)." ) - - -@pytest.mark.parametrize("submodule_dir", SUBMODULE_DIRS) -def test_required_submodule_is_initialized(submodule_dir: Path) -> None: - """The submodule's working tree is present and initialized. - - Failure means the source tree is a git clone but the submodule's - working tree hasn't been populated. Skipped when the source tree - itself isn't a git clone (e.g. an extracted release tarball), since - ``git submodule update`` cannot operate there; setups that handle - submodules in a separately-prepared tree (via - ``GIT_PYTHON_TEST_GIT_REPO_BASE``) are exempted from this check. - """ - if not (REPO_ROOT / ".git").exists(): - pytest.skip( - "Source tree is not a git clone (no .git in REPO_ROOT); submodules " - "cannot be initialized via `git submodule update` here. Setups " - "that prepare submodules in a separately-pointed tree (via " - "GIT_PYTHON_TEST_GIT_REPO_BASE) are exempted from this check." - ) - # The assertion messages below recommend `git submodule update --init - # --recursive` rather than `init-tests-after-clone.sh`, even though the - # latter is the documented entry point for first-time test setup. Two - # reasons: the script performs `git reset --hard` operations that can - # destroy local work, and #1713 showed the script itself can carry - # submodule-init regressions, in which case recommending it would lead - # developers in a circle. The direct git command is a safe minimal fix - # for this test's specific failure mode and bypasses any such regression. - assert submodule_dir.is_dir(), ( - f"Submodule working tree missing: {submodule_dir}.\n" - "Run `git submodule update --init --recursive` from the repo root." - ) - assert (submodule_dir / ".git").exists(), ( - f"Submodule directory exists but has no .git marker: {submodule_dir}.\n" - "The submodule hasn't been initialized. " - "Run `git submodule update --init --recursive` from the repo root." - ) diff --git a/test/test_fun.py b/test/test_fun.py index a456b8aab..38f506ff4 100644 --- a/test/test_fun.py +++ b/test/test_fun.py @@ -306,3 +306,7 @@ def test_linked_worktree_traversal(self, rw_dir): def test_tree_entries_from_data_with_failing_name_decode_py3(self): r = tree_entries_from_data(b"100644 \x9f\0aaa") assert r == [(b"aaa", 33188, "\udc9f")], r + + def test_tree_entries_from_bytearray(self): + r = tree_entries_from_data(bytearray(b"100644 name\0abcdefghijklmnopqrst")) + assert r == [(b"abcdefghijklmnopqrst", 33188, "name")], r diff --git a/test/test_installation.py b/test/test_installation.py index 7c82bd403..656ef9787 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -14,8 +14,9 @@ class TestInstallation(TestBase): def test_installation(self, rw_dir): venv, run = self._set_up_venv(rw_dir) - result = run([venv.pip, "install", "."]) - self._check_result(result, "Can't install project") + for project in ("./smmap", "./gitdb", "."): + result = run([venv.pip, "install", project]) + self._check_result(result, f"Can't install {project}") result = run([venv.python, "-c", "import git"]) self._check_result(result, "Self-test failed") diff --git a/test/test_repo.py b/test/test_repo.py index cfad73489..27246db88 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -948,12 +948,11 @@ def test_repo_odbtype(self): target_type = GitCmdObjectDB self.assertIsInstance(self.rorepo.odb, target_type) - def test_submodules(self): - self.assertEqual(len(self.rorepo.submodules), 1) # non-recursive - self.assertGreaterEqual(len(list(self.rorepo.iter_submodules())), 2) - - self.assertIsInstance(self.rorepo.submodule("gitdb"), Submodule) - self.assertRaises(ValueError, self.rorepo.submodule, "doesn't exist") + @with_rw_repo("c15a6e1923a14bc760851913858a3942a4193cdb", bare=True) + def test_submodules(self, rwrepo): + self.assertEqual(len(rwrepo.submodules), 1) + self.assertIsInstance(rwrepo.submodule("gitdb"), Submodule) + self.assertRaises(ValueError, rwrepo.submodule, "doesn't exist") @with_rw_repo("HEAD", bare=False) def test_submodule_update(self, rwrepo): @@ -963,11 +962,10 @@ def test_submodule_update(self, rwrepo): rwrepo._bare = False # Test submodule creation. - sm = rwrepo.submodules[0] sm = rwrepo.create_submodule( "my_new_sub", "some_path", - join_path_native(self.rorepo.working_tree_dir, sm.path), + self._small_repo_url(), ) self.assertIsInstance(sm, Submodule) diff --git a/test/test_submodule.py b/test/test_submodule.py index 265ee2ffb..f9870e08d 100644 --- a/test/test_submodule.py +++ b/test/test_submodule.py @@ -124,7 +124,7 @@ def _do_base_tests(self, rwrepo): else: with sm.config_writer() as writer: # For faster checkout, set the url to the local path. - new_smclone_path = Git.polish_url(osp.join(self.rorepo.working_tree_dir, sm.path)) + new_smclone_path = self._gitdb_repo_url() writer.set_value("url", new_smclone_path) writer.release() assert sm.config_reader().get_value("url") == new_smclone_path @@ -239,7 +239,7 @@ def _do_base_tests(self, rwrepo): # Adjust the path of the submodules module to point to the local # destination. - new_csmclone_path = Git.polish_url(osp.join(self.rorepo.working_tree_dir, sm.path, csm.path)) + new_csmclone_path = self._small_repo_url() with csm.config_writer() as writer: writer.set_value("url", new_csmclone_path) assert csm.url == new_csmclone_path @@ -491,15 +491,15 @@ def test_base_bare(self, rwrepo): @with_rw_repo(k_subm_current, bare=False) def test_root_module(self, rwrepo): # Can query everything without problems. - rm = RootModule(self.rorepo) - assert rm.module() is self.rorepo + rm = RootModule(rwrepo) + assert rm.module() is rwrepo # Try attributes. rm.binsha rm.mode rm.path assert rm.name == rm.k_root_name - assert rm.parent_commit == self.rorepo.head.commit + assert rm.parent_commit == rwrepo.head.commit rm.url rm.branch @@ -508,10 +508,6 @@ def test_root_module(self, rwrepo): with rm.config_writer(): pass - # Deep traversal yields gitdb and its nested smmap. - rsmsp = [sm.path for sm in rm.traverse()] - assert rsmsp == ["git/ext/gitdb", "gitdb/ext/smmap"] - # Cannot set the parent commit as root module's path didn't exist. self.assertRaises(ValueError, rm.set_parent_commit, "HEAD") @@ -533,7 +529,7 @@ def test_root_module(self, rwrepo): # Ensure we clone from a local source. with sm.config_writer() as writer: - writer.set_value("url", Git.polish_url(osp.join(self.rorepo.working_tree_dir, sm.path))) + writer.set_value("url", self._gitdb_repo_url()) # Dry-run does nothing. sm.update(recursive=False, dry_run=True, progress=prog) @@ -568,7 +564,7 @@ def test_root_module(self, rwrepo): # ============== nsmn = "newsubmodule" nsmp = "submrepo" - subrepo_url = Git.polish_url(osp.join(self.rorepo.working_tree_dir, rsmsp[0], rsmsp[1])) + subrepo_url = self._small_repo_url() nsm = Submodule.add(rwrepo, nsmn, nsmp, url=subrepo_url) csmadded = rwrepo.index.commit("Added submodule").hexsha # Make sure we don't keep the repo reference. nsm.set_parent_commit(csmadded) @@ -633,7 +629,7 @@ def test_root_module(self, rwrepo): # ...to the first repository. This way we have a fast checkout, and a completely # different repository at the different url. nsm.set_parent_commit(csmremoved) - nsmurl = Git.polish_url(osp.join(self.rorepo.working_tree_dir, rsmsp[0])) + nsmurl = self._gitdb_repo_url() with nsm.config_writer() as writer: writer.set_value("url", nsmurl) csmpathchange = rwrepo.index.commit("changed url") From 10834cb250d8a6553d498f16b65f9a0c796af6f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Jul 2026 08:47:53 +0000 Subject: [PATCH 1390/1392] build(deps): bump actions/setup-python from 6 to 7 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6 to 7. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/lint.yml | 2 +- .github/workflows/pythonpackage.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 79d786669..d22640951 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 with: python-version: "3.x" diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index c620e63a8..1324a9a62 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -65,7 +65,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} allow-prereleases: ${{ matrix.experimental }} From 250a65ccb539e06f5c92286154bd827920942418 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Jul 2026 08:48:06 +0000 Subject: [PATCH 1391/1392] build(deps): bump the pre-commit group with 2 updates Bumps the pre-commit group with 2 updates: [https://github.com/codespell-project/codespell](https://github.com/codespell-project/codespell) and [https://github.com/astral-sh/ruff-pre-commit](https://github.com/astral-sh/ruff-pre-commit). Updates `https://github.com/codespell-project/codespell` from v2.4.2 to 2.4.3 - [Release notes](https://github.com/codespell-project/codespell/releases) - [Commits](https://github.com/codespell-project/codespell/compare/v2.4.2...v2.4.3) Updates `https://github.com/astral-sh/ruff-pre-commit` from v0.15.20 to 0.16.0 - [Release notes](https://github.com/astral-sh/ruff-pre-commit/releases) - [Commits](https://github.com/astral-sh/ruff-pre-commit/compare/v0.15.20...v0.16.0) --- updated-dependencies: - dependency-name: https://github.com/codespell-project/codespell dependency-version: 2.4.3 dependency-type: direct:production dependency-group: pre-commit - dependency-name: https://github.com/astral-sh/ruff-pre-commit dependency-version: 0.16.0 dependency-type: direct:production dependency-group: pre-commit ... Signed-off-by: dependabot[bot] --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0ec820905..75a29d6db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,14 +2,14 @@ exclude: ^(?:gitdb|smmap)/ repos: - repo: https://github.com/codespell-project/codespell - rev: v2.4.2 + rev: v2.4.3 hooks: - id: codespell additional_dependencies: [tomli] exclude: ^test/fixtures/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.20 + rev: v0.16.0 hooks: - id: ruff-check args: ["--fix"] From 0428352c9c61c9e9ea540ebdefb52830e6ab269a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 25 Jul 2026 08:52:48 +0000 Subject: [PATCH 1392/1392] ci: use no Debian WSL identifier (as default is new enough) setup-wsl v7 removes the deprecated unversioned Debian distribution, causing every Windows matrix job to stop during environment setup. Select Debian-13 implicitly, the documented replacement for the old Debian identifier, to preserve the workflow environment while allowing the action upgrade. Co-authored-by: Eliah Kagan --- .github/workflows/pythonpackage.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 553062f2e..5126de820 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -69,7 +69,6 @@ jobs: uses: Vampire/setup-wsl@v7.0.0 with: wsl-version: 1 - distribution: Debian - name: Prepare this repo for tests run: |